Merge pull request #4 from wneessen/breaches

BreachedAccount, better HTTP handling and rate limiting
This commit is contained in:
Winni Neessen 2021-09-22 15:17:12 +02:00 committed by GitHub
commit 1606565112
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 295 additions and 42 deletions

View file

@ -3,7 +3,6 @@ package hibp
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
@ -100,25 +99,10 @@ func (b *BreachApi) Breaches(options ...BreachOption) ([]*Breach, *http.Response
queryParams := b.setBreachOpts(options...)
apiUrl := fmt.Sprintf("%s/breaches", BaseUrl)
hreq, err := b.hibp.HttpReq(http.MethodGet, apiUrl, queryParams)
hb, hr, err := b.hibp.HttpReqBody(http.MethodGet, apiUrl, queryParams)
if err != nil {
return nil, nil, err
}
hr, err := b.hibp.hc.Do(hreq)
if err != nil {
return nil, hr, err
}
if hr.StatusCode != 200 {
return nil, hr, fmt.Errorf("API responded with non HTTP-200: %s", hr.Status)
}
defer func() {
_ = hr.Body.Close()
}()
hb, err := io.ReadAll(hr.Body)
if err != nil {
return nil, hr, err
}
var breachList []*Breach
if err := json.Unmarshal(hb, &breachList); err != nil {
@ -137,26 +121,10 @@ func (b *BreachApi) BreachByName(n string, options ...BreachOption) (*Breach, *h
}
apiUrl := fmt.Sprintf("%s/breach/%s", BaseUrl, n)
hreq, err := b.hibp.HttpReq(http.MethodGet, apiUrl, queryParams)
hb, hr, err := b.hibp.HttpReqBody(http.MethodGet, apiUrl, queryParams)
if err != nil {
return nil, nil, err
}
hr, err := b.hibp.hc.Do(hreq)
if err != nil {
return nil, hr, err
}
if hr.StatusCode != 200 {
return nil, hr, fmt.Errorf("API responded with non HTTP-200: %s", hr.Status)
}
defer func() {
_ = hr.Body.Close()
}()
hb, err := io.ReadAll(hr.Body)
if err != nil {
return nil, hr, err
}
var breachDetails *Breach
if err := json.Unmarshal(hb, &breachDetails); err != nil {
@ -166,6 +134,45 @@ func (b *BreachApi) BreachByName(n string, options ...BreachOption) (*Breach, *h
return breachDetails, hr, nil
}
// DataClasses are attribute of a record compromised in a breach. This method returns a list of strings
// with all registered data classes known to HIBP
func (b *BreachApi) DataClasses() ([]string, *http.Response, error) {
apiUrl := fmt.Sprintf("%s/dataclasses", BaseUrl)
hb, hr, err := b.hibp.HttpReqBody(http.MethodGet, apiUrl, nil)
if err != nil {
return nil, nil, err
}
var dataClasses []string
if err := json.Unmarshal(hb, &dataClasses); err != nil {
return nil, hr, err
}
return dataClasses, hr, nil
}
// BreachedAccount returns a single breached site based on its name
func (b *BreachApi) BreachedAccount(a string, options ...BreachOption) ([]*Breach, *http.Response, error) {
queryParams := b.setBreachOpts(options...)
if a == "" {
return nil, nil, fmt.Errorf("no account id given")
}
apiUrl := fmt.Sprintf("%s/breachedaccount/%s", BaseUrl, a)
hb, hr, err := b.hibp.HttpReqBody(http.MethodGet, apiUrl, queryParams)
if err != nil {
return nil, nil, err
}
var breachDetails []*Breach
if err := json.Unmarshal(hb, &breachDetails); err != nil {
return nil, hr, err
}
return breachDetails, hr, nil
}
// WithDomain sets the domain filter for the breaches API
func WithDomain(d string) BreachOption {
return func(b *BreachApi) {
@ -174,6 +181,7 @@ func WithDomain(d string) BreachOption {
}
// WithoutTruncate disables the truncateResponse parameter in the breaches API
// This option only influences the BreachedAccount method
func WithoutTruncate() BreachOption {
return func(b *BreachApi) {
b.disableTrunc = true
@ -217,6 +225,9 @@ func (b *BreachApi) setBreachOpts(options ...BreachOption) map[string]string {
}
for _, opt := range options {
if opt == nil {
continue
}
opt(b)
}

View file

@ -1,6 +1,8 @@
package hibp
import (
"fmt"
"os"
"testing"
)
@ -21,6 +23,23 @@ func TestBreaches(t *testing.T) {
}
}
// TestBreachesWithNil tests the Breaches() method of the breaches API with a nil option
func TestBreachesWithNil(t *testing.T) {
hc := New()
if hc == nil {
t.Errorf("hibp client creation failed")
return
}
breachList, _, err := hc.BreachApi.Breaches(nil)
if err != nil {
t.Error(err)
}
if breachList != nil && len(breachList) <= 0 {
t.Error("breaches list returned 0 results")
}
}
// TestBreachesWithDomain tests the Breaches() method of the breaches API for a specific domain
func TestBreachesWithDomain(t *testing.T) {
testTable := []struct {
@ -133,3 +152,129 @@ func TestBreachByName(t *testing.T) {
})
}
}
// TestDataClasses tests the DataClasses() method of the breaches API
func TestDataClasses(t *testing.T) {
hc := New()
if hc == nil {
t.Errorf("hibp client creation failed")
return
}
classList, _, err := hc.BreachApi.DataClasses()
if err != nil {
t.Error(err)
}
if classList != nil && len(classList) <= 0 {
t.Error("breaches list returned 0 results")
}
}
// TestBreachedAccount tests the BreachedAccount() method of the breaches API
func TestBreachedAccount(t *testing.T) {
testTable := []struct {
testName string
accountName string
isBreached bool
moreThanOneBreach bool
}{
{"account-exists is breached once", "account-exists", true,
false},
{"multiple-breaches is breached multiple times", "multiple-breaches",
true, true},
{"opt-out is not breached", "opt-out", false, false},
}
apiKey := os.Getenv("HIBP_API_KEY")
if apiKey == "" {
t.SkipNow()
}
hc := New(WithApiKey(apiKey))
if hc == nil {
t.Error("failed to create HIBP client")
return
}
for _, tc := range testTable {
t.Run(tc.testName, func(t *testing.T) {
breachDetails, _, err := hc.BreachApi.BreachedAccount(
fmt.Sprintf("%s@hibp-integration-tests.com", tc.accountName))
if err != nil && tc.isBreached {
t.Error(err)
}
if breachDetails == nil && tc.isBreached {
t.Errorf("breach for the account %q is expected, but returned 0 results.",
tc.accountName)
}
if breachDetails != nil && !tc.isBreached {
t.Errorf("breach for the account %q is expected to be not breached, but returned breach details.",
tc.accountName)
}
if breachDetails != nil && tc.moreThanOneBreach && len(breachDetails) <= 1 {
t.Errorf("breach for the account %q is expected to be breached multiple, but returned %d breaches.",
tc.accountName, len(breachDetails))
}
if breachDetails != nil && !tc.moreThanOneBreach && len(breachDetails) > 1 {
t.Errorf("breach for the account %q is expected to be breached once, but returned %d breaches.",
tc.accountName, len(breachDetails))
}
})
}
}
// TestBreachedAccountWithoutTruncate tests the BreachedAccount() method of the breaches API with the
// truncateResponse option set to false
func TestBreachedAccountWithoutTruncate(t *testing.T) {
testTable := []struct {
testName string
accountName string
breachName string
breachDomain string
shouldFail bool
}{
{"account-exists is breached once", "account-exists", "Adobe",
"adobe.com", false},
{"multiple-breaches is breached multiple times", "multiple-breaches", "Adobe",
"adobe.com", false},
{"opt-out is not breached", "opt-out", "", "", true},
}
apiKey := os.Getenv("HIBP_API_KEY")
if apiKey == "" {
t.SkipNow()
}
hc := New(WithApiKey(apiKey), WithRateLimitNoFail())
if hc == nil {
t.Error("failed to create HIBP client")
return
}
for _, tc := range testTable {
t.Run(tc.testName, func(t *testing.T) {
breachDetails, _, err := hc.BreachApi.BreachedAccount(
fmt.Sprintf("%s@hibp-integration-tests.com", tc.accountName),
WithoutTruncate())
if err != nil && !tc.shouldFail {
t.Error(err)
return
}
if len(breachDetails) == 0 && !tc.shouldFail {
t.Errorf("breach details for account %q are expected but none were returned", tc.accountName)
return
}
if len(breachDetails) > 0 {
b := breachDetails[0]
if tc.breachName != b.Name {
t.Errorf("breach name for the account %q does not match. expected: %q, got: %q",
tc.accountName, tc.breachName, b.Name)
}
if tc.breachDomain != b.Domain {
t.Errorf("breach domain for the account %q does not match. expected: %q, got: %q",
tc.accountName, tc.breachDomain, b.Domain)
}
}
})
}
}

View file

@ -2,7 +2,7 @@ package main
import (
"fmt"
hibp "github.com/wneessen/go-hibp"
"github.com/wneessen/go-hibp"
)
func main() {

72
hibp.go
View file

@ -5,22 +5,30 @@ import (
"crypto/tls"
"fmt"
"io"
"log"
"net/http"
"net/url"
"time"
)
// Version represents the version of this package
const Version = "0.1.2"
const Version = "0.1.4"
// BaseUrl is the base URL for the majority of API calls
const BaseUrl = "https://haveibeenpwned.com/api/v3"
// DefaultUserAgent defines the default UA string for the HTTP client
// Currently the URL in the UA string is comment out, as there is a bug in the HIBP API
// not allowing multiple slashes
const DefaultUserAgent = `go-hibp v` + Version // + ` - https://github.com/wneessen/go-hibp`
// Client is the HIBP client object
type Client struct {
hc *http.Client // HTTP client to perform the API requests
to time.Duration // HTTP client timeout
ak string // HIBP API key
ua string // User agent string for the HTTP client
rlNoFail bool // Controls wether the HTTP client should fail or sleep in case the rate limiting hits
PwnedPassApi *PwnedPassApi // Reference to the PwnedPassApi API
PwnedPassApiOpts *PwnedPasswordOptions // Additional options for the PwnedPassApi API
@ -38,9 +46,13 @@ func New(options ...Option) *Client {
// Set defaults
c.to = time.Second * 5
c.PwnedPassApiOpts = &PwnedPasswordOptions{}
c.ua = DefaultUserAgent
// Set additional options
for _, opt := range options {
if opt == nil {
continue
}
opt(c)
}
@ -75,6 +87,23 @@ func WithPwnedPadding() Option {
}
}
// WithUserAgent sets a custom user agent string for the HTTP client
func WithUserAgent(a string) Option {
if a == "" {
return func(c *Client) {}
}
return func(c *Client) {
c.ua = a
}
}
// WithRateLimitNoFail let's the HTTP client sleep in case the API rate limiting hits (Defaults to fail)
func WithRateLimitNoFail() Option {
return func(c *Client) {
c.rlNoFail = true
}
}
// HttpReq performs an HTTP request to the corresponding API
func (c *Client) HttpReq(m, p string, q map[string]string) (*http.Request, error) {
u, err := url.Parse(p)
@ -106,12 +135,10 @@ func (c *Client) HttpReq(m, p string, q map[string]string) (*http.Request, error
}
hr.Header.Set("Accept", "application/json")
hr.Header.Set("User-Agent", fmt.Sprintf("go-hibp v%s - https://github.com/wneessen/go-hibp", Version))
hr.Header.Set("user-agent", c.ua)
if c.ak != "" {
hr.Header.Set("hibp-api-key", c.ak)
}
if c.PwnedPassApiOpts.WithPadding {
hr.Header.Set("Add-Padding", "true")
}
@ -119,6 +146,43 @@ func (c *Client) HttpReq(m, p string, q map[string]string) (*http.Request, error
return hr, nil
}
// HttpReqBody performs the API call to the given path and returns the response body as byte array
func (c *Client) HttpReqBody(m string, p string, q map[string]string) ([]byte, *http.Response, error) {
hreq, err := c.HttpReq(m, p, q)
if err != nil {
return nil, nil, err
}
hr, err := c.hc.Do(hreq)
if err != nil {
return nil, hr, err
}
defer func() {
_ = hr.Body.Close()
}()
hb, err := io.ReadAll(hr.Body)
if err != nil {
return nil, hr, err
}
if hr.StatusCode == 429 && c.rlNoFail {
headerDelay := hr.Header.Get("Retry-After")
delayTime, err := time.ParseDuration(headerDelay + "s")
if err != nil {
return nil, hr, err
}
log.Printf("API rate limit hit. Retrying request in %s", delayTime.String())
time.Sleep(delayTime)
return c.HttpReqBody(m, p, q)
}
if hr.StatusCode != 200 {
return nil, hr, fmt.Errorf("API responded with non HTTP-200: %s - %s", hr.Status, hb)
}
return hb, hr, nil
}
// httpClient returns a custom http client for the HIBP Client object
func httpClient(to time.Duration) *http.Client {
tlsConfig := &tls.Config{

View file

@ -1,6 +1,7 @@
package hibp
import (
"fmt"
"os"
"testing"
"time"
@ -14,6 +15,14 @@ func TestNew(t *testing.T) {
}
}
// TestNewWithNil tests the New() function with a nil option
func TestNewWithNil(t *testing.T) {
hc := New(nil)
if hc == nil {
t.Errorf("hibp client creation failed")
}
}
// TestNewWithHttpTimeout tests the New() function with the http timeout option
func TestNewWithHttpTimeout(t *testing.T) {
hc := New(WithHttpTimeout(time.Second * 10))
@ -53,3 +62,27 @@ func TestNewWithApiKey(t *testing.T) {
apiKey, hc.ak)
}
}
// TestNewWithUserAgent tests the New() function with a custom user agent
func TestNewWithUserAgent(t *testing.T) {
hc := New()
if hc == nil {
t.Errorf("hibp client creation failed")
return
}
if hc.ua != DefaultUserAgent {
t.Errorf("hibp client default user agent was not set properly. Expected %s, got: %s",
DefaultUserAgent, hc.ua)
}
custUA := fmt.Sprintf("customUA v%s", Version)
hc = New(WithUserAgent(custUA))
if hc == nil {
t.Errorf("hibp client creation failed")
return
}
if hc.ua != custUA {
t.Errorf("hibp client custom user agent was not set properly. Expected %s, got: %s",
custUA, hc.ua)
}
}