No delay value smoother

I know that the following code does not work. That's why I removed if(fade > 255).

const int ledPin = 13;
const int potPin = 7;
int potInterval = 0;
int potDuration = 0;
int fade = 0;
long preTime = 0;

void setup()
{
  pinMode(ledPin, OUTPUT);
  pinMode(potPin, INPUT);
  Serial.begin(9600);
}

void loop()
{
  unsigned long time = millis();
  potInterval = analogRead(potPin);    

  if(time - preTime > potInterval)
  {
    preTime = time;

    if(fade < 255)
    {
      fade +=5;
      analogWrite(ledPin, fade);
    }

    if(fade > 255)
    {
      fade -=5;
      analogWrite(ledPin, fade);
    }

    Serial.println(fade);
  }
}

If I'm right, when you have an if statement saying that when under 255 to climb to it by 5 and another if statement saying if at 255 to reverse that it will just stay at 255. Whereas I want it to climb to 255 and fall back to 0 after reaching it. I simply don't know how to code that without delay(), that's why I'm trying to figure it out.