Thank you very much for your patient explanation and for the debug statements. When I run the code, this is what I can see from my serial monitor:
Basically it seems to make any sense only when the rotary encoder is on the initial value of 0, in which case is shows "ready" and adds "a" when I press the key. The moment I move the encoder, it starts printing gibberish. Just in case you think it might be a problem with the encoder, it does work in every other application that I have used it in and also with the original code for this project (when the calculation part is removed).
In the meantime, nothing is shown on the OLED screen...
EDIT:
after spending a little bit of time merging my original code with yours (in a way that I can easily understand 
), I found it pretty clear that the culprit is this part of the calculation:
line 92 this is the one line that freezes everything and, if removed, allows the program to run as it should. Is it possible that this is such a complicated operation that causes the Arduino to run out of memory?...
Below my newest version of the code after making modifications based on yours
#include <Wire.h>
#include <Adafruit_GFX.h>
#include <Adafruit_SSD1306.h>
#define SCREEN_WIDTH 128 // OLED display width, in pixels
#define SCREEN_HEIGHT 64 // OLED display height, in pixels
// On an arduino UNO: A4(SDA), A5(SCL)
#define OLED_RESET 4 // Reset pin # (or -1 if sharing Arduino reset pin)
#define SCREEN_ADDRESS 0x3C ///< See datasheet for Address; 0x3D for 128x64, 0x3C for 128x32
Adafruit_SSD1306 display(SCREEN_WIDTH, SCREEN_HEIGHT, &Wire, OLED_RESET);
char InputArray[100]; // stores the selected value from char Input
char Output[100]; // stores the final result to show on screen
char Input[36] = { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' }; // shows value on screen and allows selection to store in char InputArray
char AlphaNumChart[6][6] = { { 'a', 'b', 'c', 'd', 'e', 'f' },
{ 'g', 'h', 'i', 'j', 'k', 'l' },
{ 'm', 'n', 'o', 'p', 'q', 'r' },
{ 's', 't', 'u', 'v', 'w', 'x' },
{ 'y', 'z', '0', '1', '2', '3' },
{ '4', '5', '6', '7', '8', '9' } }; // based on coordinates calculated by the formula, will select characters from this matrix to store in char Output
int num[100]={0}; //first row to store reminder, second row to store quotients
int InputLen;
int i;
int Key = 2; // key to allow selection of the value to store in char InputArray
int fadeValue1; // raw potentiometer value
int fadeValue2; // potentiometer value adjusted to select one of the 36 options from char Input
char charselect; // to show and store selected value from char Input
int InputArrayPos = 0; //this is just for word positioning on the display
int cursor; // this is just for word positioning on the display
int CursorStart; // this is just for word positioning on the display
int CursorRef; // this is just for word positioning on the display
int CursorEnd; // this is just for word positioning on the display
void setup() {
pinMode(Key, INPUT);
Serial.begin(9600);
display.begin(SSD1306_SWITCHCAPVCC, 0x3C);
display.setTextWrap(false);
display.clearDisplay();
}
void loop() {
fadeValue1 = analogRead(A0);
fadeValue2 = map(fadeValue1, 0, 1023, 0, 36);
if (fadeValue2 != 36) charselect = Input[fadeValue2];
display.clearDisplay();
displayPrint(0, 0, WHITE, 1);
display.println("Your selection:");
displayPrint(55, 15, WHITE, 4);
display.println(charselect);
if (fadeValue2 == 36) {
display.clearDisplay();
displayPrint(0, 22, WHITE, 2);
display.println("SHOW Output");
}
if ((digitalRead(Key) == 1) && (fadeValue2 != 36)) {
addCharInput();
delay(500);
}
while ((digitalRead(Key) == 1) && (fadeValue2 == 36)) {
calculateOutput();
displayOutput();
}
displayInput();
}
void calculateOutput() {
for (int i = 0; i < InputArrayPos; i++) {
int row = num[i] / 6;
int col = num[InputArrayPos - 1 - i] % 6;
Output[i] = AlphaNumChart[row][col];
}
Output[InputArrayPos] = '\0';
}
void displayOutput() {
CursorEnd = (-12 * InputLen) + display.width();
CursorStart = 0;
while (digitalRead(Key) == 1) {
if (InputLen <= 10) {
display.clearDisplay();
displayPrint(0, 0, WHITE, 1);
display.println("Your Output:");
displayPrint(0, 27, WHITE, 2);
display.println(Output);
display.display();
}
if (InputLen > 10) {
display.clearDisplay();
displayPrint(0, 0, WHITE, 1);
display.println("Your Output:");
displayPrint(CursorStart, 27, WHITE, 2);
display.println(Output);
display.display();
if (CursorStart == 0) {CursorStart = CursorStart - 2; CursorRef = 0; delay(1000);}
if ((CursorStart < 0) && (CursorRef == 0)) CursorStart = CursorStart - 2;
if (CursorStart == CursorEnd) {CursorStart = CursorStart + 2; CursorRef = 1; delay(1000);}
if ((CursorStart < 0) && (CursorRef == 1)) CursorStart = CursorStart + 2;
}
}
}
void displayPrint(int x, int y, int color, int size) {
display.setCursor(x, y);
display.setTextColor(color);
display.setTextSize(size);
}
void addCharInput() {
num[InputArrayPos] = fadeValue2; // store index for later computation
InputArray[InputArrayPos++] = Input[fadeValue2];
InputArray[InputArrayPos] = '\0';
if ( InputArrayPos >= 100 - 1 ) {
InputArrayPos = 100 - 2;
}
display.clearDisplay();
displayPrint(0, 0, WHITE, 1);
display.println("Your selection:");
displayPrint(50, 22, WHITE, 2);
display.println("OK!");
displayInput();
Serial.print("added");
Serial.print(fadeValue2);
}
void displayInput() {
int cursor = 0;
if (InputArrayPos > 10) {
cursor = (10 - InputArrayPos) * 12;
}
displayPrint(cursor, 47, WHITE, 2);
display.println(InputArray);
display.display();
}