🎭 💗 🤰🏻 पेश है गो, भाग 2: एक बाल कलाकार और विकृतियों के साथ एक इमेज ग्रैबर लिखना 🥂 👨‍🎤 🧖🏼

कार्य

हाल ही में, मैंने बात की कि कैसे मैंने एक गुप्त मिशन को अंजाम दिया और गो की मदद से मैंने "शून्यता" से एक उद्धरण डंप डाउनलोड किया । फिर से लड़ाई में शामिल होने का समय आ गया है, इस बार यह अतात की चिंता करता है, और न केवल इसलिए कि यह शून्यता के साथ गाया जाता है।

जो लोग चूक गए थे - एक समय में "थिमेटिक मीडिया" (हाँ, हब के मालिक) ने प्रोजेक्ट "रिस्पेक्ट" (एक सेक्शन के साथ ग्लैमरस इमेजबोर्ड / टॉगल स्विच का एक प्रकार) लॉन्च किया था, सरल लड़कियों और प्रोग्रामर्स को बैनरों का इंतजार करके वहाँ फुसलाया गया था और स्पष्ट रूप से उम्मीद थी कि एक साथ बढ़ें। यह एक साथ विकसित नहीं हुआ, और थोड़ी देर के बाद रेस्पेक्टा को एटैट में उत्परिवर्तित किया गया, जिसमें पहले से ही कस्टम ~~बोर्ड~~ स्ट्रीम बनाने के लिए कार्यक्षमता थी, विषयगत धाराओं के संयोजन के लिए हब, और इसी तरह। कुछ समय के लिए परियोजना विकसित हुई, लेकिन लड़कियों ने छोड़ दिया, प्रोग्रामर बने रहे, और उनमें से बहुत कम थे। अब इस जगह को लगभग छोड़ दिया गया है, प्लेग हवाएं, परित्यक्त रेलगाड़ियां, अंधेरा है और कई जीवित बचे पुराने जानवरों की केवल दुर्लभ रोएं कभी-कभी रात में html के माध्यम से कट जाती हैं।

और ठीक है, उसके साथ यह अटाट, इस अटाट के साथ। लेकिन तस्वीरों के रूप में उत्कृष्ट सामग्री का एक बहुत कुछ है, इसके लायक वॉलपेपर के साथ एक धागा! इसलिए, यह आश्चर्यजनक नहीं है कि मुख्यालय से निम्न मिशन निम्नानुसार देखा गया है:
“तत्काल ~~# 949~~ # 291 धारा से सभी छवियों को बचाने के लिए। ps। विकृत हो गया। ”

खैर, एक कार्य है - आपको काम करने की आवश्यकता है। वास्तव में, हमारे पास पहले से ही विकृत संरचनाओं में कुछ प्रकार के कॉफी चैंपियन हैं, लेकिन आप गो पर कुछ भी समझ सकते हैं! पेंट में 9 घंटे के बाद, मेरे पास निम्नलिखित हमले की योजना थी:

विकृति की डिग्री अद्भुत है, अधिकारियों को काफी होगा! आइए विचार करें कि यहां क्या है और कैसे:

"जेनरेटर" HTML फ़ाइल से पेज कोड डाउनलोड करेगा, चित्रों के url को खींचेगा और उन्हें "Balancer" में स्थानांतरित कर देगा, जो इन urls को "वर्कर्स" की एक निश्चित संख्या तक पहुंचाता है (जिनमें से प्रत्येक की अपनी छोटी सी कतार कतार है), यह सुनिश्चित करते हुए कि सभी कार्यकर्ता समान रूप से लोड किए गए हैं । खैर, "कार्यकर्ता" चित्र अपलोड करते हैं और एक छुआ बाल्कन की आंख को प्रसन्न करते हैं। यह बेमानी लग रहा है, लेकिन युद्ध के बारे में! आइए जाने:

जनक

यह सबसे आसान होगा, एक चक्र में जब हम पृष्ठों को लोड करते हैं, उन्हें पार्स करते हैं और एक चैनल में चित्रों के लम्बी यूआरएल को फीड करते हैं (मैंने पिछले लेख में चैनल, लौकी और कुछ अन्य चीजों के बारे में सतही रूप से बात की थी, इसलिए मैंने यहां दोहरा नहीं किया)।
Atat को वर्गों में विभाजित किया जाता है (जिसे "स्ट्रीम" कहा जाता है), प्रत्येक "स्ट्रीम" के एक पृष्ठ में 20 पोस्ट होते हैं। लिंक इस तरह दिखते हैं: " home.atata.com/streams/291?order=date&from=40 ", जहां 291 प्रवाह आईडी है और 40 अंत से पदों में इंडेंटेशन है (अर्थात, पहले जितना अधिक)। पोस्ट टेम्पलेट कोड जिसे हम इस तरह से देखना चाहते हैं:

<figure> <figcaption></figcaption> <a class='image' href=''> <img class='thumb' rel='' src='' /> </a> </figure>

हम url में " से " बढ़ते पृष्ठों के माध्यम से जाएंगे, छवि वर्ग के साथ लिंक खींचेंगे और उन्हें चैनल पर भेजेंगे। लेकिन कब तक? यह पता चलता है कि धारा के अंतिम पृष्ठ पर अगले बैच के पदों के लिए एक छिपी डाउनलोड बटन के रूप में एक पहचान चिह्न है (

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

), . . , , , . , , goquery (http://github.com/opesun/goquery). , : // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } } , , , ( ). - , , . , , , : " , - ?" , . PMFC (Poor Man's Flow Control " ". , ( ) ). " , , ?" ! - " " " ". ? , : ( ) - . - ( - ?). . "": // type Worker struct { urls chan string // pending int // - index int // wg sync.WaitGroup // } : Worker , struct - record . " ?" - ? Go : . , - , , : - - - . " " ( WaitGroup ) sync . : WaitGroup , , . , . , Worker'a. "" work ( , "", this. ): // func (w Worker) work(done chan Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } } sync : import ( "sync" ) : done , work() . , ( ). download() - , . . Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ). : - , : package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } , container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } , Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), : type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } , : // "": type Pool []Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p Pool) Push(x interface{}) { n := len(p) worker := x.(Worker) worker.index = n p = append(p, worker) } // : func (p Pool) Pop() interface{} { old := p n := len(old) item := old[n-1] item.index = -1 p = old[0 : n-1] return item } , interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces , heap.* Pool, "" . "container/heap" : import ( "container/heap" ) . , , , : // type Balancer struct { pool Pool // "" done chan Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg sync.WaitGroup // } , init() : // . func (b Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } } - "" . , - - ! , : make() , . , - " " ; , . init() WORKERS WORKERSCAP , : var ( WORKERS = 5 // WORKERSCAP = 5 // ) . , , ( , . , . , , ) : // func (b Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } } : // func (b Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERSWORKERSCAP { b.flowctrl <- true } } .. . : , , , . completed , : // func (b Balancer) completed(w Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } , , , . : , dispatch , , , - ? , . - download(), : // func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } - - ( ) ( , ): // : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") } - . , , os/signal , ctrl-c : import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) ! func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } } .. ( ) .. . , - ! . time.Sleep , . , "" . UPD. - ( 3 744 , 3.07Gb ) . , . pastbin.com PS , #291 () "" 10 , ( "-p=200" ) . UPD2: PPS "" : - Go, , . , , - + . , : , . , vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' sites.google.com/site/gopatterns - Go Language Patterns golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials talks.golang.org - Go , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

), . . , , , . , , goquery (http://github.com/opesun/goquery). , : // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } } , , , ( ). - , , . , , , : " , - ?" , . PMFC (Poor Man's Flow Control " ". , ( ) ). " , , ?" ! - " " " ". ? , : ( ) - . - ( - ?). . "": // type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // } : Worker , struct - record . " ?" - ? Go : . , - , , : - - - . " " ( WaitGroup ) sync . : WaitGroup , , . , . , Worker'a. "" work ( , "", this. ): // func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } } sync : import ( "sync" ) : done , work() . , ( ). download() - , . . Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ). : - , : package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } , container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } , Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), : type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } , : // "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } , interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces , heap.* Pool, "" . "container/heap" : import ( "container/heap" ) . , , , : // type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // } , init() : // . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } } - "" . , - - ! , : make() , . , - " " ; , . init() WORKERS WORKERSCAP , : var ( WORKERS = 5 // WORKERSCAP = 5 // ) . , , ( , . , . , , ) : // func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } } : // func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } .. . : , , , . completed , : // func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } , , , . : , dispatch , , , - ? , . - download(), : // func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } - - ( ) ( , ): // : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") } - . , , os/signal , ctrl-c : import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) ! func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } } .. ( ) .. . , - ! . time.Sleep , . , "" . UPD. - ( 3 744 , 3.07Gb ) . , . pastbin.com PS , #291 () "" 10 , ( "-p=200" ) . UPD2: PPS "" : - Go, , . , , - + . , : , . , vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' sites.google.com/site/gopatterns - Go Language Patterns golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials talks.golang.org - Go , !
), . . , , , . , , goquery (http://github.com/opesun/goquery). , : // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } } , , , ( ). - , , . , , , : " , - ?" , . PMFC (Poor Man's Flow Control " ". , ( ) ). " , , ?" ! - " " " ". ? , : ( ) - . - ( - ?). . "": // type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // } : Worker , struct - record . " ?" - ? Go : . , - , , : - - - . " " ( WaitGroup ) sync . : WaitGroup , , . , . , Worker'a. "" work ( , "", this. ): // func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } } sync : import ( "sync" ) : done , work() . , ( ). download() - , . . Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ). : - , : package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } , container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } , Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), : type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } , : // "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } , interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces , heap.* Pool, "" . "container/heap" : import ( "container/heap" ) . , , , : // type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // } , init() : // . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } } - "" . , - - ! , : make() , . , - " " ; , . init() WORKERS WORKERSCAP , : var ( WORKERS = 5 // WORKERSCAP = 5 // ) . , , ( , . , . , , ) : // func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } } : // func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } .. . : , , , . completed , : // func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } , , , . : , dispatch , , , - ? , . - download(), : // func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } - - ( ) ( , ): // : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") } - . , , os/signal , ctrl-c : import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) ! func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } } .. ( ) .. . , - ! . time.Sleep , . , "" . UPD. - ( 3 744 , 3.07Gb ) . , . pastbin.com PS , #291 () "" 10 , ( "-p=200" ) . UPD2: PPS "" : - Go, , . , , - + . , : , . , vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' sites.google.com/site/gopatterns - Go Language Patterns golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials talks.golang.org - Go , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

), . . , , , . , , goquery (http://github.com/opesun/goquery). , : // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } } , , , ( ). - , , . , , , : " , - ?" , . PMFC (Poor Man's Flow Control " ". , ( ) ). " , , ?" ! - " " " ". ? , : ( ) - . - ( - ?). . "": // type Worker struct { urls chan string // pending int // - index int // wg sync.WaitGroup // } : Worker , struct - record . " ?" - ? Go : . , - , , : - - - . " " ( WaitGroup ) sync . : WaitGroup , , . , . , Worker'a. "" work ( , "", this. ): // func (w Worker) work(done chan Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } } sync : import ( "sync" ) : done , work() . , ( ). download() - , . . Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ). : - , : package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } , container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } , Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), : type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } , : // "": type Pool []Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p Pool) Push(x interface{}) { n := len(p) worker := x.(Worker) worker.index = n p = append(p, worker) } // : func (p Pool) Pop() interface{} { old := p n := len(old) item := old[n-1] item.index = -1 p = old[0 : n-1] return item } , interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces , heap.* Pool, "" . "container/heap" : import ( "container/heap" ) . , , , : // type Balancer struct { pool Pool // "" done chan Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg sync.WaitGroup // } , init() : // . func (b Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } } - "" . , - - ! , : make() , . , - " " ; , . init() WORKERS WORKERSCAP , : var ( WORKERS = 5 // WORKERSCAP = 5 // ) . , , ( , . , . , , ) : // func (b Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } } : // func (b Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERSWORKERSCAP { b.flowctrl <- true } } .. . : , , , . completed , : // func (b Balancer) completed(w Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } , , , . : , dispatch , , , - ? , . - download(), : // func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } - - ( ) ( , ): // : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") } - . , , os/signal , ctrl-c : import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) ! func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } } .. ( ) .. . , - ! . time.Sleep , . , "" . UPD. - ( 3 744 , 3.07Gb ) . , . pastbin.com PS , #291 () "" 10 , ( "-p=200" ) . UPD2: PPS "" : - Go, , . , , - + . , : , . , vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' sites.google.com/site/gopatterns - Go Language Patterns golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials talks.golang.org - Go , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

 ),      .                 .  ,  ,         ,        . 
 
  ,     ,    goquery (http://github.com/opesun/goquery). ,   : 
 
 // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( //      : ENDMESSAGE = "TooLateToDieYoung" ) //          func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { //     : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } //      -   .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //..    return } } } } 
 
    
        ,        
 ,       ,             (           ).    -     ,    ,     . ,  ,   ,  : 
 
 " ,      -    ?" 
 
    ,        .        PMFC (Poor Man's Flow Control  "   ".  ,        (  ) ). 
 
 "     ,        , ?" 
 
 !         - " "  "  ".      ?           ,   : 
     (     ) -          .     -         (       -   ?). 
          .  "": 
 
 // type Worker struct { urls chan string //    pending int // -   index int //    wg *sync.WaitGroup //    } 
    :     Worker ,     struct -    record  . 
 "   ?" -  ?    Go   :            .      ,      -    ,      ,       : 
 
          -        -   -   .     " " ( WaitGroup )   sync .    : WaitGroup   ,         ,    .           ,      . 
     ,          Worker'a. 
 
   ""  work (      ,     "",  this.                 ): 
 
 //        func (w *Worker) work(done chan *Worker) { for { url := <-w.urls //   w.wg.Add(1) //    download(url) //  w.wg.Done() //     done <- w //    } } 
      sync   : 
 
 import ( "sync" ) 
    :           done ,     work() .         ,           (     ). 
  download()    -     ,   .    . 
 
   Worker Pool   ,   ,     ( Wikipedia: ( ) ),            .       " container/heap ",             ( -   ). 
 
   :            -         , : 
 
 package main import ("fmt") //  type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } 
   ,       container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : 
 
 type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } 
 ,  Push  Pop     sort.Interface ( http://golang.org/pkg/sort/#Interface ),  : 
 
 type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } 
        ,      : 
 
 //   "": type Pool []*Worker //   -         : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } //    : func (p Pool) Len() int { return len(p) } //  : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item } 
    ,   interface{} -   ,     (        ,      interface{}).     any ,          .    : http://research.swtch.com/interfaces 
   ,      heap.*   Pool,          ""    . 
    "container/heap"   : 
 
 import ( "container/heap" ) 
    .     ,  ,    ,            : 
 
 // type Balancer struct { pool Pool // ""  done chan *Worker //      requests chan string //     flowctrl chan bool //  PMFC queue int //     wg *sync.WaitGroup //    } 
         ,    init() : 
 
 // .        func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) //  Flow Control: go func() { for { b.requests <- <-in //         <-b.flowctrl //     } }() //    : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) //  heap.Push(&b.pool, w) //     } } 
       -  ""              .    ,        -      -          ! 
  ,     :  make()    ,     .   ,        -          "  " ;      ,      . 
 
  init()    WORKERS  WORKERSCAP       ,  : 
 
 var ( WORKERS = 5 //  WORKERSCAP = 5 //    ) 
      .  ,           ,     (      ,  .      ,   .    ,       , ) : 
 
 //          func (b *Balancer) balance(quit chan bool) { lastjobs := false // ,     for { select { //     : case <-quit: //     b.wg.Wait() //    .. quit <- true //..     case url := <-b.requests: //   ( flow controller) if url != ENDMESSAGE { // -      ? b.dispatch(url) //  ,    } else { lastjobs = true //    } case w := <-b.done: // ,     b.completed(w) //   if lastjobs { if w.pending == 0 { //    .. heap.Remove(&b.pool, w.index) //     } if len(b.pool) == 0 { //     //      quit <- true //        } } } } } 
   : 
 
 //    func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) //     .. w.urls <- url //..   . w.pending++ //  "".. heap.Push(&b.pool, w) //..     if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } } 
 ..    .    :         , ,  ,             . 
     completed    ,    : 
 
 //   func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } 
           ,  ,  ,             .    :   ,  dispatch           ,  ,    ,    -       ? 
   ,  . -   download(),       : 
 
 //  func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } 
 -             -   (     )       (       ,     ): 
 
 //       : var ( WORKERS = 5 //  WORKERSCAP = 5 //    ATATASTREAM = 291 //id   ATATAPOS = 0 //     IMGDIR = "img" //    ) //   : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id  ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, "  ") } 
 -          .  , ,  os/signal ,      ctrl-c   : 
 
 import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) 
      ! 
 
   
 func main() { //  flag.Parse() //   ,     if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } //    links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) //      keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) //    go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println("  ") //  : for { select { case <-keys: //    : fmt.Println("CTRL-C:    ") quit <- true //    case <-quit: //      fmt.Println(" !") return } } } 
 .. (       ) 
 
  
 
 ..    .    ,  - !    . 
 
    time.Sleep      ,                .      ,         ""     . 
 UPD.     -      ( 3 744 , 3.07Gb )    . 
 
          ,                 . 
 
     pastbin.com 
 
 PS        ,      #291 ()        ""     10   ,        (  "-p=200" )          . 
 
 UPD2: PPS        ""   :      -   Go,     ,          .       , ,        -  +          .     ,              : ,                      . 
 
  ,    
 vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' 
 sites.google.com/site/gopatterns - Go Language Patterns 
 golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials 
 talks.golang.org -      Go 
 
    , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

), . . , , , . , , goquery (http://github.com/opesun/goquery). , : // atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } } , , , ( ). - , , . , , , : " , - ?" , . PMFC (Poor Man's Flow Control " ". , ( ) ). " , , ?" ! - " " " ". ? , : ( ) - . - ( - ?). . "": // type Worker struct { urls chan string // pending int // - index int // wg sync.WaitGroup // } : Worker , struct - record . " ?" - ? Go : . , - , , : - - - . " " ( WaitGroup ) sync . : WaitGroup , , . , . , Worker'a. "" work ( , "", this. ): // func (w Worker) work(done chan Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } } sync : import ( "sync" ) : done , work() . , ( ). download() - , . . Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ). : - , : package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } } , container.Heap ( http://golang.org/pkg/container/heap/#Interface ) : type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. } , Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), : type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) } , : // "": type Pool []Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p Pool) Push(x interface{}) { n := len(p) worker := x.(Worker) worker.index = n p = append(p, worker) } // : func (p Pool) Pop() interface{} { old := p n := len(old) item := old[n-1] item.index = -1 p = old[0 : n-1] return item } , interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces , heap.* Pool, "" . "container/heap" : import ( "container/heap" ) . , , , : // type Balancer struct { pool Pool // "" done chan Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg sync.WaitGroup // } , init() : // . func (b Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } } - "" . , - - ! , : make() , . , - " " ; , . init() WORKERS WORKERSCAP , : var ( WORKERS = 5 // WORKERSCAP = 5 // ) . , , ( , . , . , , ) : // func (b Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } } : // func (b Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERSWORKERSCAP { b.flowctrl <- true } } .. . : , , , . completed , : // func (b Balancer) completed(w Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } } , , , . : , dispatch , , , - ? , . - download(), : // func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) } - - ( ) ( , ): // : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") } - . , , os/signal , ctrl-c : import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" ) ! func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } } .. ( ) .. . , - ! . time.Sleep , . , "" . UPD. - ( 3 744 , 3.07Gb ) . , . pastbin.com PS , #291 () "" 10 , ( "-p=200" ) . UPD2: PPS "" : - Go, , . , , - + . , : , . , vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism' sites.google.com/site/gopatterns - Go Language Patterns golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials talks.golang.org - Go , !

), . . , , , .

, , goquery (http://github.com/opesun/goquery). , :

// atatagrab project main.go package main import ( "fmt" // : "github.com/opesun/goquery" "strconv" ) const ( // : ENDMESSAGE = "TooLateToDieYoung" ) // func generator(out chan string, stream, start int) { for pos := start; ; pos += 20 { // : x, err := goquery.ParseUrl("http://home.atata.com/streams/" + strconv.Itoa(stream) + "?order=date&from=" + strconv.Itoa(pos)) if err == nil { // : for _, url := range x.Find("figure a.image").Attrs("href") { out <- "http://atata.com/" + url } // - .. if len(x.Find("li.last.hide")) > 0 { out <- ENDMESSAGE //.. return } } } }


,
, , ( ). - , , . , , , :

" , - ?"

, . PMFC (Poor Man's Flow Control " ". , ( ) ).

" , , ?"

! - " " " ". ? , :
( ) - . - ( - ?).
. "":

// type Worker struct { urls chan string // pending int // - index int // wg *sync.WaitGroup // }
: Worker , struct - record .
" ?" - ? Go : . , - , , :

- - - . " " ( WaitGroup ) sync . : WaitGroup , , . , .
, Worker'a.

"" work ( , "", this. ):

// func (w *Worker) work(done chan *Worker) { for { url := <-w.urls // w.wg.Add(1) // download(url) // w.wg.Done() // done <- w // } }
sync :

import ( "sync" )
: done , work() . , ( ).
download() - , . .

Worker Pool , , ( Wikipedia: ( ) ), . " container/heap ", ( - ).

: - , :

package main import ("fmt") // type Animal interface { Say() string } type Cat struct{} func (c Cat) Say() string { return "!" } type Parrot struct { name string } func (p Parrot) Say() string { return p.name + " !" } func main() { kitty := Cat{} popka := Parrot{name: ""} animals := []Animal{kitty, popka} for _, animal := range animals { fmt.Println(animal.Say()) } }
, container.Heap ( http://golang.org/pkg/container/heap/#Interface ) :

type Interface interface { sort.Interface Push(x interface{}) // add x as element Len() Pop() interface{} // remove and return element Len() - 1. }
, Push Pop sort.Interface ( http://golang.org/pkg/sort/#Interface ), :

type Interface interface { // Len is the number of elements in the collection. Len() int // Less returns whether the element with index i should sort // before the element with index j. Less(i, j int) bool // Swap swaps the elements with indexes i and j. Swap(i, j int) }
, :

// "": type Pool []*Worker // - : func (p Pool) Less(i, j int) bool { return p[i].pending < p[j].pending } // : func (p Pool) Len() int { return len(p) } // : func (p Pool) Swap(i, j int) { if i >= 0 && i < len(p) && j >= 0 && j < len(p) { p[i], p[j] = p[j], p[i] p[i].index, p[j].index = i, j } } // : func (p *Pool) Push(x interface{}) { n := len(*p) worker := x.(*Worker) worker.index = n *p = append(*p, worker) } // : func (p *Pool) Pop() interface{} { old := *p n := len(old) item := old[n-1] item.index = -1 *p = old[0 : n-1] return item }
, interface{} - , ( , interface{}). any , . : http://research.swtch.com/interfaces
, heap.* Pool, "" .
"container/heap" :

import ( "container/heap" )
. , , , :

// type Balancer struct { pool Pool // "" done chan *Worker // requests chan string // flowctrl chan bool // PMFC queue int // wg *sync.WaitGroup // }
, init() :

// . func (b *Balancer) init(in chan string) { b.requests = make(chan string) b.flowctrl = make(chan bool) b.done = make(chan *Worker) b.wg = new(sync.WaitGroup) // Flow Control: go func() { for { b.requests <- <-in // <-b.flowctrl // } }() // : heap.Init(&b.pool) for i := 0; i < WORKERS; i++ { w := &Worker{ urls: make(chan string, WORKERSCAP), index: 0, pending: 0, wg: b.wg, } go w.work(b.done) // heap.Push(&b.pool, w) // } }
- "" . , - - !
, : make() , . , - " " ; , .

init() WORKERS WORKERSCAP , :

var ( WORKERS = 5 // WORKERSCAP = 5 // )
. , , ( , . , . , , ) :

// func (b *Balancer) balance(quit chan bool) { lastjobs := false // , for { select { // : case <-quit: // b.wg.Wait() // .. quit <- true //.. case url := <-b.requests: // ( flow controller) if url != ENDMESSAGE { // - ? b.dispatch(url) // , } else { lastjobs = true // } case w := <-b.done: // , b.completed(w) // if lastjobs { if w.pending == 0 { // .. heap.Remove(&b.pool, w.index) // } if len(b.pool) == 0 { // // quit <- true // } } } } }
:

// func (b *Balancer) dispatch(url string) { w := heap.Pop(&b.pool).(*Worker) // .. w.urls <- url //.. . w.pending++ // "".. heap.Push(&b.pool, w) //.. if b.queue++; b.queue < WORKERS*WORKERSCAP { b.flowctrl <- true } }
.. . : , , , .
completed , :

// func (b *Balancer) completed(w *Worker) { w.pending-- heap.Remove(&b.pool, w.index) heap.Push(&b.pool, w) if b.queue--; b.queue == WORKERS*WORKERSCAP-1 { b.flowctrl <- true } }
, , , . : , dispatch , , , - ?
, . - download(), :

// func download(url string) { fileName := IMGDIR + "/" + url[strings.LastIndex(url, "/")+1:] output, err := os.Create(fileName) defer output.Close() response, err := http.Get(url) if err != nil { fmt.Println("Error while downloading", url, "-", err) return } defer response.Body.Close() io.Copy(output, response.Body) }
- - ( ) ( , ):

// : var ( WORKERS = 5 // WORKERSCAP = 5 // ATATASTREAM = 291 //id ATATAPOS = 0 // IMGDIR = "img" // ) // : func init() { flag.IntVar(&WORKERS, "w", WORKERS, " ") flag.IntVar(&ATATASTREAM, "s", ATATASTREAM, "id ") flag.IntVar(&ATATAPOS, "p", ATATAPOS, " ") flag.StringVar(&IMGDIR, "d", IMGDIR, " ") }
- . , , os/signal , ctrl-c :

import ( "io" "os" "os/signal" "sync" "flag" "fmt" "container/heap" "strconv" "strings" "github.com/opesun/goquery" "net/http" )
!


func main() { // flag.Parse() // , if err := os.MkdirAll(IMGDIR, 666); err != nil { panic(err) } // links := make(chan string) quit := make(chan bool) b := new(Balancer) b.init(links) // keys keys := make(chan os.Signal, 1) signal.Notify(keys, os.Interrupt) // go b.balance(quit) go generator(links, ATATASTREAM, ATATAPOS) fmt.Println(" ") // : for { select { case <-keys: // : fmt.Println("CTRL-C: ") quit <- true // case <-quit: // fmt.Println(" !") return } } }
.. ( )



.. . , - ! .

time.Sleep , . , "" .
UPD. - ( 3 744 , 3.07Gb ) .

, .

pastbin.com

PS , #291 () "" 10 , ( "-p=200" ) .

UPD2: PPS "" : - Go, , . , , - + . , : , .

,
vimeo.com/49718712 - Rob Pike - 'Concurrency Is Not Parallelism'
sites.google.com/site/gopatterns - Go Language Patterns
golangtutorials.blogspot.ru/2011/05/table-of-contents.html - GoLang Tutorials
talks.golang.org - Go

, !

पेश है गो, भाग 2: एक बाल कलाकार और विकृतियों के साथ एक इमेज ग्रैबर लिखना

कार्य

जनक

More articles: