Ah no, the pump is not reversible, it just pumps either way (but worse with wrong polarity).
Yes, D4 is set up as input pullup. Hang on, I am nearly done with the code.
Here's the full code, now that you started pointing to D4 
#include <Wire.h> // required for i2c
#include <Adafruit_GFX.h>
#include <Adafruit_SSD1306.h>
// #include <Adafruit_AM2320.h>
// this is the display object, the constant is just
// needed for initiation:
#define OLED_RESET 4
Adafruit_SSD1306 display(OLED_RESET);
// this is the temp and humid sensor:
// #define Adafruit_AM2320 am2320 = Adafruit_AM2320()
const int moistPin1 = A0; // the pin where the moisture sensor is connected
const int moistPin2 = A1;
const int pumpSwitchPin = 5; // the pin of the switch/relay that switches on the pump
const int pumpIndicatorLedPin = LED_BUILTIN; // select the pin for the LED
int moistValue1 = 500;
int moistValue2 = 500;
int moistAvg = 0;
const int rotarySwPin = 4;
const int rotaryDtPin = 3;
const int rotaryClkPin = 2;
int clkStateCurrent = LOW;
int clkStateLast = LOW;
int lastButtonPressMillis = millis();
int lastButtonReleaseMillis = millis();
int moistWanted = 500;
const int DOUBLE_CLICK_THRESHOLD = 333; // button click within ... milliseconds
const int LONG_CLICK_THRESHOLD = 3000; // millis required to pass for long click
const int moistThresholdOn = 10; // difference in reading points that the
// reading has to be **lower** than desired
// to switch the pump ON
const int moistThresholdOff = 10; // difference in reading points that the
// reading has to be **higher** than the
// wanted to switch the pump OFF
bool pumpNeeded = false;
bool asleep = false;
// Only the dt is observed constantly. This happens here:
void dtFunc() {
// stop any other interrupts during this process:
detachInterrupt(digitalPinToInterrupt(rotaryDtPin));
detachInterrupt(digitalPinToInterrupt(rotarySwPin));
clkStateCurrent = digitalRead(rotaryClkPin);
if (clkStateLast == LOW && clkStateCurrent == HIGH) {
if (digitalRead(rotaryDtPin) == HIGH) {
moistWanted = max(moistWanted - 1, 0);
} else {
moistWanted = min(moistWanted + 1, 1024);
}
}
clkStateLast = clkStateCurrent;
attachInterrupt(digitalPinToInterrupt(rotaryDtPin), dtFunc, CHANGE);
attachInterrupt(digitalPinToInterrupt(rotarySwPin), swFunc, CHANGE);
}
void swFunc() {
int currentMillis = millis();
detachInterrupt(digitalPinToInterrupt(rotarySwPin));
detachInterrupt(digitalPinToInterrupt(rotaryDtPin));
if (asleep) {
}
if (digitalRead(rotarySwPin) == LOW) {
if (currentMillis - lastButtonPressMillis < DOUBLE_CLICK_THRESHOLD) {
doubleClickAction();
}
lastButtonPressMillis = currentMillis;
} else if (digitalRead(rotarySwPin) == HIGH) {
lastButtonReleaseMillis = currentMillis;
}
attachInterrupt(digitalPinToInterrupt(rotarySwPin), swFunc, CHANGE);
attachInterrupt(digitalPinToInterrupt(rotaryDtPin), dtFunc, CHANGE);
}
int getMoistAvg(int moist1, int moist2) {
return(round((moist1 + moist2) / 2));
}
void doubleClickAction() {
Serial.println("double Click happened");
}
void setup() {
pinMode(pumpIndicatorLedPin, OUTPUT);
pinMode(pumpSwitchPin, OUTPUT);
pinMode(rotarySwPin, INPUT_PULLUP);
pinMode(rotaryDtPin, INPUT);
pinMode(rotaryClkPin, INPUT);
attachInterrupt(digitalPinToInterrupt(rotaryDtPin), dtFunc, CHANGE);
attachInterrupt(digitalPinToInterrupt(rotarySwPin), swFunc, CHANGE);
analogReference(EXTERNAL);
Serial.begin(9600);
// init display:
display.begin(SSD1306_SWITCHCAPVCC, 0x3C);
// init temp and humid sensor:
// am2320.begin()
}
void displayMain() {
display.clearDisplay();
display.setTextColor(WHITE);
display.setTextSize(1);
display.setCursor(0, 0);
display.print("Moist 1, 2: ");
display.print(moistValue1);
display.print(", ");
display.print(moistValue2);
display.setCursor(0, 10);
display.print("Avg: ");
display.print(moistAvg);
display.print(" | want: ");
display.print(moistWanted);
display.setCursor(0, 20);
display.print(millis());
}
void displayCalibration() {
delay(100);
}
int readMoisture(int pin, int prevVal) {
int idx;
int reading;
int readingSum = 0;
int n_iter = 5;
for (idx = 0; idx < n_iter; idx ++) {
readingSum += analogRead(pin);
delay(10);
}
reading = round(readingSum / n_iter);
if (abs(prevVal - reading) > 5) {
return(reading);
}
return(prevVal);
}
void printDataToSerial() {
// prints the available data in JSON format to Serial
Serial.print("{");
Serial.print("\"moisture-1\":");
Serial.print(moistValue1);
Serial.print(",");
Serial.print("\"moisture-2\":");
Serial.print(moistValue2);
Serial.print("\"moisture-avg\":");
Serial.print(moistAvg);
Serial.print(",");
Serial.print("\"temperature\":");
Serial.print(0.0);
Serial.print(",");
Serial.print("\"humidity\":");
Serial.print(0.0);
Serial.print(",");
Serial.print("\"moist-wanted\":");
Serial.print(moistWanted);
Serial.print("}");
Serial.print('\n');
}
void pumpActivity() {
// note that high values mean *dry* for capacitive sensors!
if (moistAvg - moistWanted > moistThresholdOn && !pumpNeeded) {
pumpNeeded = true;
digitalWrite(pumpIndicatorLedPin, HIGH);
digitalWrite(pumpSwitchPin, HIGH);
} else if (moistWanted - moistAvg > moistThresholdOff && pumpNeeded) {
pumpNeeded = false;
digitalWrite(pumpIndicatorLedPin, LOW);
digitalWrite(pumpSwitchPin, LOW);
}
}
void mainRun() {
// main function if awake
if (Serial.available() > 0) {
while (Serial.available() > 0) {
if (Serial.readStringUntil('\n') == "print_data") {
printDataToSerial();
}
}
}
// read the value from the sensor:
moistValue1 = readMoisture(moistPin1, moistValue1); // has 50ms delay
// moistValue2 = readMoisture(moistPin2, moistValue2); // has 50ms delay
moistAvg = getMoistAvg(moistValue1, moistValue2);
displayMain();
display.display();
// supervise pump:
pumpActivity();
if (millis() - lastButtonReleaseMillis > LONG_CLICK_THRESHOLD && digitalRead(rotarySwPin) == LOW) {
Serial.println("long click happened");
}
}
void loop() {
//delay(10000);
if (asleep) {
// LowPower.deepSleep(1000 * 60 * 30); // sleep for 30 Minutes
} else {
mainRun();
}
}
