diff --git a/common/Makefile b/common/Makefile
index 33c606abb..abca82072 100644
--- a/common/Makefile
+++ b/common/Makefile
@@ -32,7 +32,6 @@ COBJS-y += command.o
 COBJS-y += exports.o
 COBJS-$(CONFIG_SYS_HUSH_PARSER) += hush.o
 COBJS-y += s_record.o
-COBJS-$(CONFIG_SERIAL_MULTI) += serial.o
 COBJS-y += xyzModem.o
 COBJS-y += cmd_disk.o
 
diff --git a/drivers/serial/Makefile b/drivers/serial/Makefile
index afaddeaa2..a8b500b22 100644
--- a/drivers/serial/Makefile
+++ b/drivers/serial/Makefile
@@ -25,6 +25,10 @@ include $(TOPDIR)/config.mk
 
 LIB	:= $(obj)libserial.o
 
+ifndef CONFIG_SPL_BUILD
+COBJS-$(CONFIG_SERIAL_MULTI) += serial.o
+endif
+
 COBJS-$(CONFIG_ALTERA_UART) += altera_uart.o
 COBJS-$(CONFIG_ALTERA_JTAG_UART) += altera_jtag_uart.o
 COBJS-$(CONFIG_ARM_DCC) += arm_dcc.o
diff --git a/common/serial.c b/drivers/serial/serial.c
similarity index 100%
rename from common/serial.c
rename to drivers/serial/serial.c