Merge pull request #4 from wneessen/breaches

BreachedAccount, better HTTP handling and rate limiting

breach.go

@@ -3,7 +3,6 @@ package hibp
 import (
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -100,25 +99,10 @@ func (b *BreachApi) Breaches(options ...BreachOption) ([]*Breach, *http.Response
 	queryParams := b.setBreachOpts(options...)
 	apiUrl := fmt.Sprintf("%s/breaches", BaseUrl)
 
-	hreq, err := b.hibp.HttpReq(http.MethodGet, apiUrl, queryParams)
+	hb, hr, err := b.hibp.HttpReqBody(http.MethodGet, apiUrl, queryParams)
 	if err != nil {
 		return nil, nil, err
 	}
-	hr, err := b.hibp.hc.Do(hreq)
-	if err != nil {
-		return nil, hr, err
-	}
-	if hr.StatusCode != 200 {
-		return nil, hr, fmt.Errorf("API responded with non HTTP-200: %s", hr.Status)
-	}
-	defer func() {
-		_ = hr.Body.Close()
-	}()
-
-	hb, err := io.ReadAll(hr.Body)
-	if err != nil {
-		return nil, hr, err
-	}
 
 	var breachList []*Breach
 	if err := json.Unmarshal(hb, &breachList); err != nil {
@@ -137,26 +121,10 @@ func (b *BreachApi) BreachByName(n string, options ...BreachOption) (*Breach, *h
 	}
 
 	apiUrl := fmt.Sprintf("%s/breach/%s", BaseUrl, n)
-
+	hb, hr, err := b.hibp.HttpReqBody(http.MethodGet, apiUrl, queryParams)
-	hreq, err := b.hibp.HttpReq(http.MethodGet, apiUrl, queryParams)
 	if err != nil {
 		return nil, nil, err
 	}
-	hr, err := b.hibp.hc.Do(hreq)
-	if err != nil {
-		return nil, hr, err
-	}
-	if hr.StatusCode != 200 {
-		return nil, hr, fmt.Errorf("API responded with non HTTP-200: %s", hr.Status)
-	}
-	defer func() {
-		_ = hr.Body.Close()
-	}()
-
-	hb, err := io.ReadAll(hr.Body)
-	if err != nil {
-		return nil, hr, err
-	}
 
 	var breachDetails *Breach
 	if err := json.Unmarshal(hb, &breachDetails); err != nil {
@@ -166,6 +134,45 @@ func (b *BreachApi) BreachByName(n string, options ...BreachOption) (*Breach, *h
 	return breachDetails, hr, nil
 }
 
+// DataClasses are attribute of a record compromised in a breach. This method returns a list of strings
+// with all registered data classes known to HIBP
+func (b *BreachApi) DataClasses() ([]string, *http.Response, error) {
+	apiUrl := fmt.Sprintf("%s/dataclasses", BaseUrl)
+	hb, hr, err := b.hibp.HttpReqBody(http.MethodGet, apiUrl, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var dataClasses []string
+	if err := json.Unmarshal(hb, &dataClasses); err != nil {
+		return nil, hr, err
+	}
+
+	return dataClasses, hr, nil
+}
+
+// BreachedAccount returns a single breached site based on its name
+func (b *BreachApi) BreachedAccount(a string, options ...BreachOption) ([]*Breach, *http.Response, error) {
+	queryParams := b.setBreachOpts(options...)
+
+	if a == "" {
+		return nil, nil, fmt.Errorf("no account id given")
+	}
+
+	apiUrl := fmt.Sprintf("%s/breachedaccount/%s", BaseUrl, a)
+	hb, hr, err := b.hibp.HttpReqBody(http.MethodGet, apiUrl, queryParams)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var breachDetails []*Breach
+	if err := json.Unmarshal(hb, &breachDetails); err != nil {
+		return nil, hr, err
+	}
+
+	return breachDetails, hr, nil
+}
+
 // WithDomain sets the domain filter for the breaches API
 func WithDomain(d string) BreachOption {
 	return func(b *BreachApi) {
@@ -174,6 +181,7 @@ func WithDomain(d string) BreachOption {
 }
 
 // WithoutTruncate disables the truncateResponse parameter in the breaches API
+// This option only influences the BreachedAccount method
 func WithoutTruncate() BreachOption {
 	return func(b *BreachApi) {
 		b.disableTrunc = true
@@ -217,6 +225,9 @@ func (b *BreachApi) setBreachOpts(options ...BreachOption) map[string]string {
 	}
 
 	for _, opt := range options {
+		if opt == nil {
+			continue
+		}
 		opt(b)
 	}
 

breach_test.go

@@ -1,6 +1,8 @@
 package hibp
 
 import (
+	"fmt"
+	"os"
 	"testing"
 )
 
@@ -21,6 +23,23 @@ func TestBreaches(t *testing.T) {
 	}
 }
 
+// TestBreachesWithNil tests the Breaches() method of the breaches API with a nil option
+func TestBreachesWithNil(t *testing.T) {
+	hc := New()
+	if hc == nil {
+		t.Errorf("hibp client creation failed")
+		return
+	}
+
+	breachList, _, err := hc.BreachApi.Breaches(nil)
+	if err != nil {
+		t.Error(err)
+	}
+	if breachList != nil && len(breachList) <= 0 {
+		t.Error("breaches list returned 0 results")
+	}
+}
+
 // TestBreachesWithDomain tests the Breaches() method of the breaches API for a specific domain
 func TestBreachesWithDomain(t *testing.T) {
 	testTable := []struct {
@@ -133,3 +152,129 @@ func TestBreachByName(t *testing.T) {
 		})
 	}
 }
+
+// TestDataClasses tests the DataClasses() method of the breaches API
+func TestDataClasses(t *testing.T) {
+	hc := New()
+	if hc == nil {
+		t.Errorf("hibp client creation failed")
+		return
+	}
+
+	classList, _, err := hc.BreachApi.DataClasses()
+	if err != nil {
+		t.Error(err)
+	}
+	if classList != nil && len(classList) <= 0 {
+		t.Error("breaches list returned 0 results")
+	}
+}
+
+// TestBreachedAccount tests the BreachedAccount() method of the breaches API
+func TestBreachedAccount(t *testing.T) {
+	testTable := []struct {
+		testName          string
+		accountName       string
+		isBreached        bool
+		moreThanOneBreach bool
+	}{
+		{"account-exists is breached once", "account-exists", true,
+			false},
+		{"multiple-breaches is breached multiple times", "multiple-breaches",
+			true, true},
+		{"opt-out is not breached", "opt-out", false, false},
+	}
+
+	apiKey := os.Getenv("HIBP_API_KEY")
+	if apiKey == "" {
+		t.SkipNow()
+	}
+	hc := New(WithApiKey(apiKey))
+	if hc == nil {
+		t.Error("failed to create HIBP client")
+		return
+	}
+
+	for _, tc := range testTable {
+		t.Run(tc.testName, func(t *testing.T) {
+			breachDetails, _, err := hc.BreachApi.BreachedAccount(
+				fmt.Sprintf("%s@hibp-integration-tests.com", tc.accountName))
+			if err != nil && tc.isBreached {
+				t.Error(err)
+			}
+
+			if breachDetails == nil && tc.isBreached {
+				t.Errorf("breach for the account %q is expected, but returned 0 results.",
+					tc.accountName)
+			}
+			if breachDetails != nil && !tc.isBreached {
+				t.Errorf("breach for the account %q is expected to be not breached, but returned breach details.",
+					tc.accountName)
+			}
+			if breachDetails != nil && tc.moreThanOneBreach && len(breachDetails) <= 1 {
+				t.Errorf("breach for the account %q is expected to be breached multiple, but returned %d breaches.",
+					tc.accountName, len(breachDetails))
+			}
+			if breachDetails != nil && !tc.moreThanOneBreach && len(breachDetails) > 1 {
+				t.Errorf("breach for the account %q is expected to be breached once, but returned %d breaches.",
+					tc.accountName, len(breachDetails))
+			}
+		})
+	}
+}
+
+// TestBreachedAccountWithoutTruncate tests the BreachedAccount() method of the breaches API with the
+// truncateResponse option set to false
+func TestBreachedAccountWithoutTruncate(t *testing.T) {
+	testTable := []struct {
+		testName     string
+		accountName  string
+		breachName   string
+		breachDomain string
+		shouldFail   bool
+	}{
+		{"account-exists is breached once", "account-exists", "Adobe",
+			"adobe.com", false},
+		{"multiple-breaches is breached multiple times", "multiple-breaches", "Adobe",
+			"adobe.com", false},
+		{"opt-out is not breached", "opt-out", "", "", true},
+	}
+
+	apiKey := os.Getenv("HIBP_API_KEY")
+	if apiKey == "" {
+		t.SkipNow()
+	}
+	hc := New(WithApiKey(apiKey), WithRateLimitNoFail())
+	if hc == nil {
+		t.Error("failed to create HIBP client")
+		return
+	}
+
+	for _, tc := range testTable {
+		t.Run(tc.testName, func(t *testing.T) {
+			breachDetails, _, err := hc.BreachApi.BreachedAccount(
+				fmt.Sprintf("%s@hibp-integration-tests.com", tc.accountName),
+				WithoutTruncate())
+			if err != nil && !tc.shouldFail {
+				t.Error(err)
+				return
+			}
+			if len(breachDetails) == 0 && !tc.shouldFail {
+				t.Errorf("breach details for account %q are expected but none were returned", tc.accountName)
+				return
+			}
+
+			if len(breachDetails) > 0 {
+				b := breachDetails[0]
+				if tc.breachName != b.Name {
+					t.Errorf("breach name for the account %q does not match. expected: %q, got: %q",
+						tc.accountName, tc.breachName, b.Name)
+				}
+				if tc.breachDomain != b.Domain {
+					t.Errorf("breach domain for the account %q does not match. expected: %q, got: %q",
+						tc.accountName, tc.breachDomain, b.Domain)
+				}
+			}
+		})
+	}
+}

examples/breaches/breach-by-name.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	hibp "github.com/wneessen/go-hibp"
+	"github.com/wneessen/go-hibp"
 )
 
 func main() {

hibp.go

@@ -5,22 +5,30 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"net/url"
 	"time"
 )
 
 // Version represents the version of this package
-const Version = "0.1.2"
+const Version = "0.1.4"
 
 // BaseUrl is the base URL for the majority of API calls
 const BaseUrl = "https://haveibeenpwned.com/api/v3"
 
+// DefaultUserAgent defines the default UA string for the HTTP client
+// Currently the URL in the UA string is comment out, as there is a bug in the HIBP API
+// not allowing multiple slashes
+const DefaultUserAgent = `go-hibp v` + Version // + ` - https://github.com/wneessen/go-hibp`
+
 // Client is the HIBP client object
 type Client struct {
 	hc       *http.Client  // HTTP client to perform the API requests
 	to       time.Duration // HTTP client timeout
 	ak       string        // HIBP API key
+	ua       string        // User agent string for the HTTP client
+	rlNoFail bool          // Controls wether the HTTP client should fail or sleep in case the rate limiting hits
 
 	PwnedPassApi     *PwnedPassApi         // Reference to the PwnedPassApi API
 	PwnedPassApiOpts *PwnedPasswordOptions // Additional options for the PwnedPassApi API
@@ -38,9 +46,13 @@ func New(options ...Option) *Client {
 	// Set defaults
 	c.to = time.Second * 5
 	c.PwnedPassApiOpts = &PwnedPasswordOptions{}
+	c.ua = DefaultUserAgent
 
 	// Set additional options
 	for _, opt := range options {
+		if opt == nil {
+			continue
+		}
 		opt(c)
 	}
 
@@ -75,6 +87,23 @@ func WithPwnedPadding() Option {
 	}
 }
 
+// WithUserAgent sets a custom user agent string for the HTTP client
+func WithUserAgent(a string) Option {
+	if a == "" {
+		return func(c *Client) {}
+	}
+	return func(c *Client) {
+		c.ua = a
+	}
+}
+
+// WithRateLimitNoFail let's the HTTP client sleep in case the API rate limiting hits (Defaults to fail)
+func WithRateLimitNoFail() Option {
+	return func(c *Client) {
+		c.rlNoFail = true
+	}
+}
+
 // HttpReq performs an HTTP request to the corresponding API
 func (c *Client) HttpReq(m, p string, q map[string]string) (*http.Request, error) {
 	u, err := url.Parse(p)
@@ -106,12 +135,10 @@ func (c *Client) HttpReq(m, p string, q map[string]string) (*http.Request, error
 	}
 
 	hr.Header.Set("Accept", "application/json")
-	hr.Header.Set("User-Agent", fmt.Sprintf("go-hibp v%s - https://github.com/wneessen/go-hibp", Version))
+	hr.Header.Set("user-agent", c.ua)
-
 	if c.ak != "" {
 		hr.Header.Set("hibp-api-key", c.ak)
 	}
-
 	if c.PwnedPassApiOpts.WithPadding {
 		hr.Header.Set("Add-Padding", "true")
 	}
@@ -119,6 +146,43 @@ func (c *Client) HttpReq(m, p string, q map[string]string) (*http.Request, error
 	return hr, nil
 }
 
+// HttpReqBody performs the API call to the given path and returns the response body as byte array
+func (c *Client) HttpReqBody(m string, p string, q map[string]string) ([]byte, *http.Response, error) {
+	hreq, err := c.HttpReq(m, p, q)
+	if err != nil {
+		return nil, nil, err
+	}
+	hr, err := c.hc.Do(hreq)
+	if err != nil {
+		return nil, hr, err
+	}
+	defer func() {
+		_ = hr.Body.Close()
+	}()
+
+	hb, err := io.ReadAll(hr.Body)
+	if err != nil {
+		return nil, hr, err
+	}
+
+	if hr.StatusCode == 429 && c.rlNoFail {
+		headerDelay := hr.Header.Get("Retry-After")
+		delayTime, err := time.ParseDuration(headerDelay + "s")
+		if err != nil {
+			return nil, hr, err
+		}
+		log.Printf("API rate limit hit. Retrying request in %s", delayTime.String())
+		time.Sleep(delayTime)
+		return c.HttpReqBody(m, p, q)
+	}
+
+	if hr.StatusCode != 200 {
+		return nil, hr, fmt.Errorf("API responded with non HTTP-200: %s - %s", hr.Status, hb)
+	}
+
+	return hb, hr, nil
+}
+
 // httpClient returns a custom http client for the HIBP Client object
 func httpClient(to time.Duration) *http.Client {
 	tlsConfig := &tls.Config{

hibp_test.go

@@ -1,6 +1,7 @@
 package hibp
 
 import (
+	"fmt"
 	"os"
 	"testing"
 	"time"
@@ -14,6 +15,14 @@ func TestNew(t *testing.T) {
 	}
 }
 
+// TestNewWithNil tests the New() function with a nil option
+func TestNewWithNil(t *testing.T) {
+	hc := New(nil)
+	if hc == nil {
+		t.Errorf("hibp client creation failed")
+	}
+}
+
 // TestNewWithHttpTimeout tests the New() function with the http timeout option
 func TestNewWithHttpTimeout(t *testing.T) {
 	hc := New(WithHttpTimeout(time.Second * 10))
@@ -53,3 +62,27 @@ func TestNewWithApiKey(t *testing.T) {
 			apiKey, hc.ak)
 	}
 }
+
+// TestNewWithUserAgent tests the New() function with a custom user agent
+func TestNewWithUserAgent(t *testing.T) {
+	hc := New()
+	if hc == nil {
+		t.Errorf("hibp client creation failed")
+		return
+	}
+	if hc.ua != DefaultUserAgent {
+		t.Errorf("hibp client default user agent was not set properly. Expected %s, got: %s",
+			DefaultUserAgent, hc.ua)
+	}
+
+	custUA := fmt.Sprintf("customUA v%s", Version)
+	hc = New(WithUserAgent(custUA))
+	if hc == nil {
+		t.Errorf("hibp client creation failed")
+		return
+	}
+	if hc.ua != custUA {
+		t.Errorf("hibp client custom user agent was not set properly. Expected %s, got: %s",
+			custUA, hc.ua)
+	}
+}