69 lines
1.3 KiB
Go
69 lines
1.3 KiB
Go
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
|
|
|
package ecload
|
|
|
|
import (
|
|
"fmt"
|
|
"github.com/PuerkitoBio/goquery"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"sync"
|
|
)
|
|
|
|
// Download a html page from an url (must be UTF-8) and convert it to a goquery document.
|
|
func fetchDocument(url string) (*goquery.Document, error) {
|
|
res, err := http.Get(url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer res.Body.Close()
|
|
|
|
if res.StatusCode != 200 {
|
|
return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromReader(res.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return doc, nil
|
|
}
|
|
|
|
// Download a file.
|
|
func downloadToFile(filename string, dir string, pageUrl string, wg sync.WaitGroup) error {
|
|
wg.Add(1)
|
|
defer wg.Done()
|
|
|
|
fullpath := path.Join(dir, filename)
|
|
|
|
out, err := os.Create(fullpath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer out.Close()
|
|
|
|
res, err := http.Get(pageUrl)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
if res.StatusCode != 200 {
|
|
return fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
|
|
}
|
|
|
|
_, err = io.Copy(out, res.Body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|