package main

import (
   "fmt"
   "net/http"
   "sync"

   "golang.org/x/net/html"
)

func main() {
   urls := []string{
      "http://example.com/",
      "http://example.org/",
      "http://example.net/",
   }

   fetchAll(urls)
}

func fetchAll(urls []string) {
   var wg sync.WaitGroup

   for _, url := range urls {
      wg.Add(1)
      go fetch(&wg, url)
   }

   wg.Wait()
}

func fetch(wg *sync.WaitGroup, url string) {
   defer wg.Done()

   res, err := http.Get(url)
   if err != nil {
      fmt.Printf("Error fetching: %s\n", url)
      return
   }
   defer res.Body.Close()

   doc, err := html.Parse(res.Body)
   if err != nil {
      fmt.Printf("Error parsing: %s\n", url)
   }

   title := extractTitle(doc)
   fmt.Printf("Title of %s: %s\n", url, title)
}
//這里我們只做簡單的解析標(biāo)題，可以根據(jù)實(shí)際應(yīng)用場景進(jìn)行操作
func extractTitle(doc *html.Node) string {
   var title string

   traverseNodes := func(n *html.Node) {
      if n.Type == html.ElementNode && n.Data == "title" {
         title = n.FirstChild.Data
      }
      for c := n.FirstChild; c != nil; c = c.NextSibling {
         traverseNodes(c)
      }
   }

   traverseNodes(doc)

   return title
}

// Import packages
package main

import (
   "fmt"
   "sync"
   "net/http"
   "io/ioutil"
   "regexp"
   "time"
)

// Maximum number of working goroutines
const MaxWorkNum = 10

// URL channel
var UrlChannel = make(chan string, MaxWorkNum)
// Results channel
var ResultsChannel = make(chan string, MaxWorkNum)

func GenerateUrlProducer(seedUrl string) {
   go func() {
      UrlChannel <- seedUrl
   }()
}

func GenerateWorkers() {
   var wg sync.WaitGroup
   // Limit the number of working goroutines
   for i := 0; i < MaxWorkNum; i++ {
      wg.Add(1)
      go func(i int) {
         defer wg.Done()
         for {
            url, ok := <- UrlChannel
            if !ok {
               return
            }
            newUrls, err := fetch(url)
            if err != nil {
               fmt.Printf("Worker %d: %v\n", i, err)
               return
            }
            for _, newUrl := range newUrls {
               UrlChannel <- newUrl
            }
            ResultsChannel <- url
         }
      }(i)
   }
   wg.Wait()
}

func fetch(url string) ([]string, error) {
   resp, err := http.Get(url)
   if err != nil {
      return nil, err
   }
   body, _ := ioutil.ReadAll(resp.Body)
   urlRegexp := regexp.MustCompile(`http[s]?://[^"'\s]+`)
   newUrls := urlRegexp.FindAllString(string(body), -1)
   resp.Body.Close()
   time.Sleep(time.Second) // to prevent IP being blocked
   return newUrls, nil
}

func ResultsConsumer() {
   for {
      url, ok := <- ResultsChannel
      if !ok {
         return
      }
      fmt.Println("Fetched:", url)
   }
}

func main() {
   go GenerateUrlProducer("http://example.com")
   go GenerateWorkers()
   ResultsConsumer()
}