"Cannot convert varchar to numeric" or "string or binary data would be truncated"

I am trying to do an insert and I am receiving the error when trying to add in an additional clause on my where statement using SS#. The table I am copying from is all varchar columns and it works properly until I add in the line to check that the SS#'s match. I understand getting the convert error so I tried using the convert function in the subquery and then I get the truncate error.

--Grab the max record_serial_number from DESTINATION
declare @rsnOwner numeric
set @rsnOwner = (select max(RECORD_SERIAL_NUMBER)
                from DESTINATION)

--Insert values from TEMP to DESTINATION
insert into DESTINATION
    (
        RECORD_SERIAL_NUMBER,
        AGENT_NUMBER,
        SS_NO,
        LAST_NAME_OWNER,
        FIRST_NAME_OWNER,
        TITLE,
        BIRTHDAY,
        HOME_STREET_ADDRESS,
        HOME_TOWN,
        HOME_STATE,
        HOME_ZIP_CODE,
        HOME_PHONE,
        BANKRUPT,
        MODIFY_DATE,
        MODIFY_TIME,
        USER_ID,
        SHARE_PCT,
        CELL_PHONE,
        CONTACT_TYPE,
        CONTACT_DESC,
        CH_APPROVED,
        CH_DATE,
        ADDRESS_VERIFY,
        VERIFY_DATE
    )
Select
        @rsnOwner + ROW_NUMBER() over (order by agent_number),
        convert(numeric, AGENT_NUMBER),
        convert(numeric, replace(ss_num,'-','')), --remove dashes and convert
        CONTACT_LAST_NAME,
        CONTACT_FIRST_NAME,
        CONTACT_TITLE,
        case when isdate(DOB)=1 then convert(datetime, '19'+right(DOB, 2)+left(DOB, 2)+substring(DOB,3,2)) else null end,   --rearrange the 6 digit date to 8 digit in YYYYMMDD format
        ADDRESS1,
        CITY,
        STATE,
        case when isnumeric(ZIP_CODE)=1 then convert(numeric, ZIP_CODE) else null end,
        case when isnumeric (CONTACT_PHONE_NUM)=1 then replace(convert(numeric, CONTACT_PHONE_NUM),'-','') else null end, --remove dashes and convert
        ')', --BANKRUPT
        convert(datetime, GetDate()) as MODIFY_DATE,
        replace(Convert (varchar(8), GetDate(), 108),':',''),
        'dbo_update', --USERID
        cast(PERCENT_OWNERSHIP as numeric (5,2)) as PERCENT_OWNERSHIP,
        case when isnumeric(CONTACT_CELL_NUM)=1 then replace(convert(numeric, CONTACT_CELL_NUM),'-','') else null end, --remove dashes and convert
        TYPE,
        TYPE_DESC,
        CH_APPROVED,
        case when isdate(CH_DATE)=1 then convert(datetime, '20'+right(CH_DATE, 2)+left(CH_DATE, 2)+substring(CH_DATE,3,2)) else null end, --rearrange the 6 digit date to 8 digit in YYYYMMDD format
        ADD_VERIFICATION,
        case when isdate(ADD_VERIFICATION_DATE)=1 then convert(datetime, '20'+right(ADD_VERIFICATION_DATE, 2)+left(ADD_VERIFICATION_DATE, 2)+substring(ADD_VERIFICATION_DATE,3,2)) else null end --rearrange the 6 digit date to 8 digit in YYYYMMDD format
from TEMP as tmp
where NOT EXISTS(select NULL
                 from DESTINATION as dest
                 where tmp.AGENT_NUMBER = dest.AGENT_NUMBER
                 and
                 convert(numeric, replace(SS_NUM,'-','')) = dest.SS_NO
                 ) 

The tables do not have primary keys and it is a many to many relationship using either agent_number or SS# but the two of them together is unique. I also tried creating a variable for SS_NUM so I can use it in both the select and the where clause but apparently I can't do that and would have to create variables for everything in my select.