I'm having a problems getting news headlines from a NY Times API. I get connected OK but I cannot consistantly retrieve the JSON string it sends. At one point I printed out the string on the serial monitor and then compared it to receiving that string in a web browser. They matched and the string length was something like 58,000 bytes - pretty large. When I repeat the request I usually get partial string like 30,000 bytes. I assume that there is a buffer problem on the incoming data. I've seen message hints that BUFFERSIZE can be changed in WifiClientSecure.h, but that doesn't exist nor does a method called setBufferSizes(writge, read).
Here is the function I've created and have tried reading the incomming data byte by byte and also using readString(). Neither works and readString() doesn't seem to appear in WifiSecureClient.cpp, but it compiles without an error.
Any suggestions appreciated.
Thanks,
Theron Wierenga
void getNewsFromInternet()
{
// New key from NY Times Jan 2024
// https://api.nytimes.com/svc/topstories/v2/science.json?api-key=d7GNBsyUwMa7tGDAGLtYZFjLHB7SqTbM
const char* server = "api.nytimes.com";
WiFiClientSecure client;
client.setInsecure();
const int httpPort = 443; // 80 is for HTTP / 443 is for HTTPS!
int httpCode;
Serial.println("\nStarting connection to server...");
httpCode = client.connect(server, 443);
Serial.println(httpCode);
if (httpCode == 0)
Serial.println("Connection failed!");
else {
Serial.println("Connected to server!");
// Make a HTTP request:
client.println("GET /svc/topstories/v2/science.json?api-key=d7GNBsyUwMa7tGDAGLtYZFjLHB7SqTbM HTTP/1.1");
client.println("Host: api.nytimes.com");
client.println("Connection: close");
client.println("User-Agent: ArduinoWiFi/1.0"); // This is an example User-Agent; you can set it to anything relevant
client.println();
while (client.connected()) {
String line = client.readStringUntil('\n');
if (line == "\r") {
Serial.println("headers received");
break;
}
}
// if there are incoming bytes available
// from the server, read them and print them:
while (client.available()) {
news_payload = client.readString();
// c = client.read();
// Serial.write(c);
// news_payload += c;
}
Serial.println(news_payload);
Serial.print("news_payload length = ");
Serial.println(news_payload.length());
client.stop();
}
}
the point was to not save the json in memory for the moment - just to see if you get it all from the web request or if you already lost data because of the way the web client works.
The evidence is that I get different string lengths on what comes down and looking at the end of the string it doesn't have a correct json ending.
Here's my latest attempt, I've changed it to read into a char array.
// Retrieve news headlines from newsapi.org
// *****************************************************************************************************************
void getNewsFromInternet()
{
const char* server = "newsapi.org";
WiFiClientSecure client;
client.setInsecure();
const int httpPort = 443; // 80 is for HTTP / 443 is for HTTPS!
int httpCode, count = 0;
news_payload = " ";
Serial.println("\nStarting connection to server...");
httpCode = client.connect(server, 443);
Serial.println(httpCode);
if (httpCode == 0)
Serial.println("Connection failed!");
else {
Serial.println("Connected to server!");
// Make a HTTP request:
client.println("GET /v2/top-headlines?country=us&apiKey=myKey HTTP/1.1");
client.println("Host: newsapi.org");
client.println("Connection: close");
client.println("User-Agent: ArduinoWiFi/1.0"); // This is an example User-Agent; you can set it to anything relevant
client.println();
while (client.connected()) {
String line = client.readStringUntil('\n');
if (line == "\r") {
Serial.println("headers received");
break;
}
}
// if there are incoming bytes available
// from the server, read them and print them:
count = 0;
while (client.available()) {
n_payload[count++] = client.read();
// news_payload += c;
}
n_payload[count] = 0x00;
// Convert to String
news_payload = String(n_payload);
Serial.print("news_payload = ");
Serial.println(news_payload);
Serial.println(news_payload.length());
Serial.println(count);
client.stop();
}
}
