Browse Source

Tidy up doco

Ryan Armstrong 3 years ago
parent
commit
98d648b740
6 changed files with 112 additions and 93 deletions
  1. 17 13
      client.go
  2. 14 4
      client_test.go
  3. 4 3
      error.go
  4. 29 28
      grab.go
  5. 24 37
      request.go
  6. 24 8
      response.go

+ 17 - 13
client.go

@@ -12,8 +12,7 @@ import (
 	"time"
 )
 
-// A Client is an HTTP file transfer client. Its zero value is a usable client
-// that uses http.Client defaults.
+// A Client is a file download client.
 //
 // Clients are safe for concurrent use by multiple goroutines.
 type Client struct {
@@ -28,8 +27,7 @@ type Client struct {
 	UserAgent string
 }
 
-// NewClient returns a new file transfer Client, using default transport
-// configuration.
+// NewClient returns a new file download Client, using default configuration.
 func NewClient() *Client {
 	return &Client{
 		UserAgent: "grab",
@@ -41,11 +39,11 @@ func NewClient() *Client {
 	}
 }
 
-// DefaultClient is the default client and is used by Get.
+// DefaultClient is the default client and is used by all Get convenience
+// functions.
 var DefaultClient = NewClient()
 
 // CancelRequest cancels an in-flight request by closing its connection.
-// CancelRequest should only be called after a Response is returned.
 func (c *Client) CancelRequest(req *Request) {
 	if t, ok := c.HTTPClient.Transport.(*http.Transport); ok {
 		t.CancelRequest(req.HTTPRequest)
@@ -58,6 +56,10 @@ func (c *Client) CancelRequest(req *Request) {
 //
 // An error is returned if caused by client policy (such as CheckRedirect), or
 // if there was an HTTP protocol error.
+//
+// Do is a synchronous, blocking operation which returns only once a download
+// request is completed or fails. For non-blocking operations which enable the
+// monitoring of transfers in process, see DoAsync and DoBatch.
 func (c *Client) Do(req *Request) (*Response, error) {
 	// prepare request with HEAD request
 	resp, err := c.do(req)
@@ -81,11 +83,11 @@ func (c *Client) Do(req *Request) (*Response, error) {
 // The Response is sent via the returned channel and the channel closed as soon
 // as the HTTP/1.1 GET request has been served; before the file transfer begins.
 //
-// The Response may then be used to gauge the progress of the file transfer
+// The Response may then be used to monitor the progress of the file transfer
 // while it is in process.
 //
 // Any error which occurs during the file transfer will be set in the returned
-// Response.Error field at the time which it occurs.
+// Response.Error field as soon as the Response.IsComplete method returns true.
 func (c *Client) DoAsync(req *Request) <-chan *Response {
 	r := make(chan *Response, 1)
 	go func() {
@@ -104,17 +106,19 @@ func (c *Client) DoAsync(req *Request) <-chan *Response {
 }
 
 // DoBatch executes multiple requests with the given number of workers and
-// returns a channel to receive the file transfer response contexts. The channel
-// is closed once all responses have been received.
+// immediately returns a channel to receive the Responses as they become
+// available. Excess requests are queued until a worker becomes available. The
+// channel is closed once all responses have been sent.
 //
 // If zero is given as the worker count, one worker will be created for each
-// given request.
+// given request and all requests will start at the same time.
 //
 // Each response is sent through the channel once the request is initiated via
-// HTTP GET or an error has occurred but before the file transfer begins.
+// HTTP GET or an error has occurred, but before the file transfer begins.
 //
 // Any error which occurs during any of the file transfers will be set in the
-// associated Response.Error field.
+// associated Response.Error field as soon as the Response.IsComplete method
+// returns true.
 func (c *Client) DoBatch(workers int, reqs ...*Request) <-chan *Response {
 	// TODO: enable cancelling of batch jobs
 

+ 14 - 4
client_test.go

@@ -176,6 +176,8 @@ func TestChecksums(t *testing.T) {
 	testChecksum(t, 1048576, "fbbab289f7f94b25736c58be46a994c441fd02552cc6022352e3d86d2fab7c82", false)
 }
 
+// testSize executes a request and asserts that the file size for the downloaded
+// file does or does not match the expected size.
 func testSize(t *testing.T, url string, size uint64, match bool) {
 	req, _ := NewRequest(url)
 	req.Filename = ".testSize-mismatch-head"
@@ -201,6 +203,7 @@ func testSize(t *testing.T, url string, size uint64, match bool) {
 	}
 }
 
+// TestSize exeuctes a number of size tests via testSize.
 func TestSize(t *testing.T) {
 	size := uint64(32768)
 
@@ -220,6 +223,7 @@ func TestSize(t *testing.T) {
 	// TODO: testSize(t, ts.URL+fmt.Sprintf("?nocl&size=%d", size), size, false)
 }
 
+// TestAutoResume tests segmented downloading of a large file.
 func TestAutoResume(t *testing.T) {
 	segs := 8
 	size := 1048576
@@ -294,6 +298,8 @@ func TestAutoResume(t *testing.T) {
 	}
 }
 
+// TestBatch executes multiple requests simultaneously and validates the
+// responses.
 func TestBatch(t *testing.T) {
 	tests := 256
 	size := 32768
@@ -302,7 +308,7 @@ func TestBatch(t *testing.T) {
 
 	// create requests
 	done := make(chan *Response, 0)
-	reqs := make(Requests, tests)
+	reqs := make([]*Request, tests)
 	for i := 0; i < len(reqs); i++ {
 		reqs[i], _ = NewRequest(ts.URL + fmt.Sprintf("/request_%d?size=%d", i, size))
 		reqs[i].Label = fmt.Sprintf("Test %d", i+1)
@@ -337,14 +343,16 @@ func TestBatch(t *testing.T) {
 	}
 }
 
+// TestCancel validates that a request can be successfully cancelled before a
+// file transfer starts.
 func TestCancel(t *testing.T) {
 	client := NewClient()
 
-	// slow request (3000ms)
+	// slow request
 	req, _ := NewRequest(ts.URL + "/.testCancel?sleep=2000")
 	ch := client.DoAsync(req)
 
-	// sleep and cancel
+	// sleep and cancel before request is served
 	time.Sleep(500 * time.Millisecond)
 	client.CancelRequest(req)
 
@@ -357,10 +365,12 @@ func TestCancel(t *testing.T) {
 	}
 }
 
+// TestCancelInProcess validates that a request can be successfully cancelled
+// after a file transfer has started.
 func TestCancelInProcess(t *testing.T) {
 	client := NewClient()
 
-	// slow request (3000ms)
+	// large file request
 	req, _ := NewRequest(ts.URL + "/.testCancelInProcess?size=134217728")
 	done := make(chan *Response)
 	req.NotifyOnClose = done

+ 4 - 3
error.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 )
 
+// error codes
 const (
 	errBadLength = iota
 	errNoFilename
@@ -42,15 +43,15 @@ func isErrorType(err error, code int) bool {
 }
 
 // IsContentLengthMismatch returns a boolean indicating whether the error is
-// known to report that a HTTP request response indicated that the requested
-// file is not the expected length.
+// known to report that a HTTP response indicated that the requested file is not
+// the expected length.
 func IsContentLengthMismatch(err error) bool {
 	return isErrorType(err, errBadLength)
 }
 
 // IsNoFilename returns a boolean indicating whether the error is known to
 // report that a destination filename could not be determined from the
-// Content-Disposition headers of a HTTP response or the request URL's path.
+// Content-Disposition headers of a HTTP response or the requested URL path.
 func IsNoFilename(err error) bool {
 	return isErrorType(err, errNoFilename)
 }

+ 29 - 28
grab.go

@@ -18,16 +18,6 @@ partial file. If the server does not support resumed downloads, the file will be
 retransferred in its entirety. If the file is already complete, grab will return
 successfully.
 
-If any HTTP response is one of the following redirect codes, grab follows the
-redirect, up to a maximum of 10 redirects:
-
-	301 (Moved Permanently)
-	302 (Found)
-	303 (See Other)
-	307 (Temporary Redirect)
-
-An error is returned if there were too many redirects or if there was an HTTP
-protocol error.
 */
 package grab
 
@@ -35,10 +25,18 @@ import (
 	"os"
 )
 
-// Get tranfers a file from the specified source URL to the given destination
-// path and returns the completed Response context.
+// Get sends a file transfer request and returns a file transfer response
+// context, following policy (e.g. redirects, cookies, auth) as configured on
+// the client's HTTPClient.
+//
+// An error is returned if caused by client policy (such as CheckRedirect), or
+// if there was an HTTP protocol error.
 //
-// To make a request with custom headers, use NewRequest and DefaultClient.Do.
+// Get is a synchronous, blocking operation which returns only once a download
+// request is completed or fails. For non-blocking operations which enable the
+// monitoring of transfers in process, see GetAsync, GetBatch or use a Client.
+//
+// Get is a wrapper for DefaultClient.Do.
 func Get(dst, src string) (*Response, error) {
 	// init client and request
 	req, err := NewRequest(src)
@@ -52,20 +50,19 @@ func Get(dst, src string) (*Response, error) {
 	return DefaultClient.Do(req)
 }
 
-// GetAsync tranfers a file from the specified source URL to the given
-// destination path and returns the Response context.
+// GetAsync sends a file transfer request and returns a channel to receive the
+// file transfer response context.
 //
-// The Response is returned as soon as a HTTP/1.1 HEAD request has completed to
-// determine the size of the requested file and supported server features.
+// The Response is sent via the returned channel and the channel closed as soon
+// as the HTTP/1.1 GET request has been served; before the file transfer begins.
 //
-// The Response may then be used to gauge the progress of the file transfer
+// The Response may then be used to monitor the progress of the file transfer
 // while it is in process.
 //
-// If an error occurs while initializing the request, it will be returned
-// immediately. Any error which occurs during the file transfer will instead be
-// set on the returned Response at the time which it occurs.
+// Any error which occurs during the file transfer will be set in the returned
+// Response.Error field as soon as the Response.IsComplete method returns true.
 //
-// To make a request with custom headers, use NewRequest and DefaultClient.Do.
+// GetAsync is a wrapper for DefaultClient.DoAsync.
 func GetAsync(dst, src string) (<-chan *Response, error) {
 	// init client and request
 	req, err := NewRequest(src)
@@ -80,20 +77,24 @@ func GetAsync(dst, src string) (<-chan *Response, error) {
 }
 
 // GetBatch executes multiple requests with the given number of workers and
-// returns a channel to receive the file transfer response contexts. The channel
-// is closed once all responses have been received.
+// immediately returns a channel to receive the Responses as they become
+// available. Excess requests are queued until a worker becomes available. The
+// channel is closed once all responses have been sent.
 //
 // GetBatch requires that the destination path is an existing directory. If not,
 // an error is returned which may be identified with IsBadDestination.
 //
 // If zero is given as the worker count, one worker will be created for each
-// given request.
+// given request and all requests will start at the same time.
 //
 // Each response is sent through the channel once the request is initiated via
-// HTTP GET or an error has occurred but before the file transfer begins.
+// HTTP GET or an error has occurred, but before the file transfer begins.
 //
 // Any error which occurs during any of the file transfers will be set in the
-// associated Response.Error field.
+// associated Response.Error field as soon as the Response.IsComplete method
+// returns true.
+//
+// GetBatch is a wrapper for DefaultClient.GetBatch.
 func GetBatch(workers int, dst string, sources ...string) (<-chan *Response, error) {
 	// check that dst is an existing directory
 	fi, err := os.Stat(dst)
@@ -106,7 +107,7 @@ func GetBatch(workers int, dst string, sources ...string) (<-chan *Response, err
 	}
 
 	// build slice of request
-	reqs := make(Requests, len(sources))
+	reqs := make([]*Request, len(sources))
 	for i := 0; i < len(sources); i++ {
 		req, err := NewRequest(sources[i])
 		if err != nil {

+ 24 - 37
request.go

@@ -11,57 +11,50 @@ import (
 	"net/url"
 )
 
-// A Request represents an HTTP file transfer request to be sent by a client.
+// A Request represents an HTTP file transfer request to be sent by a Client.
 type Request struct {
-	// Label is an arbitrary string which may used to identify a request when it
-	// is returned, attached to a Response.
+	// Label is an arbitrary string which may used to label a Request with a
+	// user friendly name.
 	Label string
 
-	// Tag is an arbitrary interface which may be used to relate a request to
-	// other data when it is returned, attached to a Response.
+	// Tag is an arbitrary interface which may be used to relate a Request to
+	// other data.
 	Tag interface{}
 
-	// HTTPRequest specifies the HTTP request to be sent to the remote server
-	// to initiate a file transfer. It includes request configuration such as
-	// URL, protocol version, HTTP method, request headers and authentication.
+	// HTTPRequest specifies the http.Request to be sent to the remote server to
+	// initiate a file transfer. It includes request configuration such as URL,
+	// protocol version, HTTP method, request headers and authentication.
 	HTTPRequest *http.Request
 
 	// Filename specifies the path where the file transfer will be stored in
 	// local storage.
 	//
-	// If the given Filename is a directory, the file transfer will be stored in
-	// that directory and the file's name will be determined using
-	// Content-Disposition headers in the server's response or from the base
-	// name in the request URL.
-	//
 	// An empty string means the transfer will be stored in the current working
 	// directory.
 	Filename string
 
-	// RemoveOnError specifies that any completed download should be deleted if
-	// it fails checksum validation.
-	RemoveOnError bool
-
 	// Size specifies the expected size of the file transfer if known. If the
 	// server response size does not match, the transfer is cancelled and an
 	// error returned.
 	Size uint64
 
-	// Hash specifies the hashing algorithm that should be used to compute the
-	// trasferred file's checksum value.
+	// Hash specifies the hashing algorithm that will be used to compute the
+	// checksum value of the transferred file.
+	//
+	// If Checksum or Hash is nil, no checksum validation occurs.
 	Hash hash.Hash
 
-	// Checksum specifies the checksum value which should be compared with the
-	// checksum value computed for the transferred file by the given hashing
-	// algorithm.
+	// Checksum specifies the expected checksum value of the transferred file.
 	//
-	// If the checksum values do not match, the file is deleted and an error
-	// returned.
+	// If Checksum or Hash is nil, no checksum validation occurs.
 	Checksum []byte
 
-	// NotifyOnClose specifies a channel that will notified with a pointer to
-	// the Response when the transfer is completed, either successfully or with
-	// an error.
+	// RemoveOnError specifies that any completed download should be deleted if
+	// it fails checksum validation.
+	RemoveOnError bool
+
+	// NotifyOnClose specifies a channel that will notified when the requested
+	// transfer is completed, either successfully or with an error.
 	NotifyOnClose chan<- *Response
 
 	// notifyOnCloseInternal is the same as NotifyOnClose but for private
@@ -69,11 +62,8 @@ type Request struct {
 	notifyOnCloseInternal chan *Response
 }
 
-// Requests is a slice of pointers to Request structs.
-type Requests []*Request
-
-// NewRequest returns a new file transfer Request given a URL, suitable for use
-// with client.Do.
+// NewRequest returns a new file transfer Request suitable for use with
+// Client.Do.
 func NewRequest(urlStr string) (*Request, error) {
 	// create http request
 	req, err := http.NewRequest("GET", urlStr, nil)
@@ -91,13 +81,10 @@ func (c *Request) URL() *url.URL {
 	return c.HTTPRequest.URL
 }
 
-// SetChecksum sets the request's checksum value and hashing algorithm to use
+// SetChecksum sets the expected checksum value and hashing algorithm to use
 // when validating a completed file transfer.
 //
-// If the checksum values do not match, the file is deleted and an error
-// returned.
-//
-// Supported hashing algoriths are supported:
+// The following hashing algorithms are supported:
 //	md5
 //	sha1
 //	sha256

+ 24 - 8
response.go

@@ -21,24 +21,30 @@ type Response struct {
 	Request *Request
 
 	// HTTPResponse specifies the HTTP response received from the remote server.
-	// The response's Body is nil (having already been consumed).
+	//
+	// The response Body should not be used as it will be consumed and closed by
+	// grab.
 	HTTPResponse *http.Response
 
 	// Filename specifies the path where the file transfer is stored in local
 	// storage.
 	Filename string
 
-	// Size specifies the total size of the file transfer.
+	// Size specifies the total expected size of the file transfer.
 	Size uint64
 
-	// Error specifies any error that may have occurred during the file transfer
-	// that created this response.
+	// Error specifies any error that may have occurred during the file
+	// transfer.
+	//
+	// This should not be read until IsComplete returns true.
 	Error error
 
 	// Start specifies the time at which the file transfer started.
 	Start time.Time
 
 	// End specifies the time at which the file transfer completed.
+	//
+	// This should not be read until IsComplete returns true.
 	End time.Time
 
 	// DidResume specifies that the file transfer resumed a previously
@@ -65,19 +71,21 @@ func (c *Response) IsComplete() bool {
 }
 
 // BytesTransferred returns the number of bytes which have already been
-// downloaded.
+// downloaded, including any data used to resume a previous download.
 func (c *Response) BytesTransferred() uint64 {
 	return atomic.LoadUint64(&c.bytesTransferred)
 }
 
 // Progress returns the ratio of bytes which have already been downloaded over
-// the total content length as a fraction of 1.00.
+// the total file size as a fraction of 1.00.
+//
+// Multiply the returned value by 100 to return the percentage completed.
 func (c *Response) Progress() float64 {
 	if c.Size == 0 {
 		return 0
 	}
 
-	return float64(atomic.LoadUint64(&c.bytesTransferred)) / float64(c.Size)
+	return float64(c.BytesTransferred()) / float64(c.Size)
 }
 
 // Duration returns the duration of a file transfer. If the transfer is in
@@ -95,6 +103,7 @@ func (c *Response) Duration() time.Duration {
 // AverageBytesPerSecond returns the average bytes transferred per second over
 // the duration of the file transfer.
 func (c *Response) AverageBytesPerSecond() float64 {
+	// TODO: Fix bad AvgBytesPerSec reading for resumed downloads
 	return float64(c.BytesTransferred()) / c.Duration().Seconds()
 }
 
@@ -124,6 +133,7 @@ func (c *Response) copy() error {
 		// break when finished
 		if err == io.EOF {
 			// download is ready for checksum validation
+			c.HTTPResponse.Body.Close()
 			c.writer.Close()
 			complete = true
 		}
@@ -156,17 +166,23 @@ func (c *Response) copy() error {
 		}
 	}
 
+	// finalize
 	return c.close(nil)
 }
 
 // close finalizes the response context
 func (c *Response) close(err error) error {
-	// close any file handle
+	// close writer
 	if c.writer != nil {
 		c.writer.Close()
 		c.writer = nil
 	}
 
+	// close response body
+	if c.HTTPResponse != nil && c.HTTPResponse.Body != nil {
+		c.HTTPResponse.Body.Close()
+	}
+
 	// set result error (if any)
 	c.Error = err