diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 8cf1638a08e1..f1e19ded9b5d 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -1025,12 +1025,13 @@ CONFIG_SND_AUDIO_GRAPH_CARD2=m
 CONFIG_HID_MULTITOUCH=m
 CONFIG_I2C_HID_ACPI=m
 CONFIG_I2C_HID_OF=m
+CONFIG_USB_CONN_GPIO=y
 CONFIG_USB=y
 CONFIG_USB_OTG=y
 CONFIG_USB_XHCI_HCD=y
 CONFIG_USB_XHCI_PCI_RENESAS=m
 CONFIG_USB_XHCI_RZV2M=y
-CONFIG_USB_XHCI_TEGRA=y
+CONFIG_USB_XHCI_TEGRA=m
 CONFIG_USB_BRCMSTB=m
 CONFIG_USB_EHCI_HCD=y
 CONFIG_USB_EHCI_EXYNOS=y
@@ -1511,7 +1512,7 @@ CONFIG_PHY_ROCKCHIP_TYPEC=y
 CONFIG_PHY_SAMSUNG_UFS=y
 CONFIG_PHY_UNIPHIER_USB2=y
 CONFIG_PHY_UNIPHIER_USB3=y
-CONFIG_PHY_TEGRA_XUSB=y
+CONFIG_PHY_TEGRA_XUSB=m
 CONFIG_PHY_AM654_SERDES=m
 CONFIG_PHY_J721E_WIZ=m
 CONFIG_ARM_CCI_PMU=m
diff --git a/arch/arm64/configs/tegra_prod_defconfig b/arch/arm64/configs/tegra_prod_defconfig
index 29aa66c869b9..c4332d885580 100644
--- a/arch/arm64/configs/tegra_prod_defconfig
+++ b/arch/arm64/configs/tegra_prod_defconfig
@@ -1016,12 +1016,13 @@ CONFIG_SND_AUDIO_GRAPH_CARD2=m
 CONFIG_HID_MULTITOUCH=m
 CONFIG_I2C_HID_ACPI=m
 CONFIG_I2C_HID_OF=m
+CONFIG_USB_CONN_GPIO=y
 CONFIG_USB=y
 CONFIG_USB_OTG=y
 CONFIG_USB_XHCI_HCD=y
 CONFIG_USB_XHCI_PCI_RENESAS=m
 CONFIG_USB_XHCI_RZV2M=y
-CONFIG_USB_XHCI_TEGRA=y
+CONFIG_USB_XHCI_TEGRA=m
 CONFIG_USB_BRCMSTB=m
 CONFIG_USB_EHCI_HCD=y
 CONFIG_USB_EHCI_EXYNOS=y
@@ -1501,7 +1502,7 @@ CONFIG_PHY_ROCKCHIP_TYPEC=y
 CONFIG_PHY_SAMSUNG_UFS=y
 CONFIG_PHY_UNIPHIER_USB2=y
 CONFIG_PHY_UNIPHIER_USB3=y
-CONFIG_PHY_TEGRA_XUSB=y
+CONFIG_PHY_TEGRA_XUSB=m
 CONFIG_PHY_AM654_SERDES=m
 CONFIG_PHY_J721E_WIZ=m
 CONFIG_ARM_CCI_PMU=m