Du har inte använt THEN sökord och END IF .
Prova detta:
SET @ExistedRefID = NULL;
SELECT ReferenceID INTO @ExistedRefID
FROM filesrefrences
WHERE RefrenceKey= pRefrenceKey;
IF (@ExistedRefID IS NULL) THEN
INSERT INTO filesrefrences(RefrenceKey, RefrenceCount)
VALUES(pRefrenceKey,1);
SELECT LAST_INSERT_ID();
ELSE
UPDATE filesrefrences
SET RefrenceCount= RefrenceCount+1
WHERE ReferenceID= @ExistedRefID;
SELECT @ExistedRefID;
END IF;