diff --git a/client.go b/client.go
index 2e48358..f65766d 100644
--- a/client.go
+++ b/client.go
@@ -751,6 +751,7 @@ func (c *Client) SetTLSPolicy(policy TLSPolicy) {
 func (c *Client) SetTLSPortPolicy(policy TLSPolicy) {
 	if c.port == DefaultPort {
 		c.port = DefaultPortTLS
+		c.fallbackPort = 0
 
 		if policy == TLSOpportunistic {
 			c.fallbackPort = DefaultPort